{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 608
    },
    "id": "6yQnx-q9ET1H",
    "outputId": "58cf800c-ce8b-44c5-b4dd-767b6fe11887"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting transformers\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/19/22/aff234f4a841f8999e68a7a94bdd4b60b4cebcfeca5d67d61cd08c9179de/transformers-3.3.1-py3-none-any.whl (1.1MB)\n",
      "\u001b[K     |████████████████████████████████| 1.1MB 8.8MB/s \n",
      "\u001b[?25hCollecting sacremoses\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/7d/34/09d19aff26edcc8eb2a01bed8e98f13a1537005d31e95233fd48216eed10/sacremoses-0.0.43.tar.gz (883kB)\n",
      "\u001b[K     |████████████████████████████████| 890kB 30.2MB/s \n",
      "\u001b[?25hCollecting tokenizers==0.8.1.rc2\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/80/83/8b9fccb9e48eeb575ee19179e2bdde0ee9a1904f97de5f02d19016b8804f/tokenizers-0.8.1rc2-cp36-cp36m-manylinux1_x86_64.whl (3.0MB)\n",
      "\u001b[K     |████████████████████████████████| 3.0MB 58.9MB/s \n",
      "\u001b[?25hCollecting sentencepiece!=0.1.92\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/d4/a4/d0a884c4300004a78cca907a6ff9a5e9fe4f090f5d95ab341c53d28cbc58/sentencepiece-0.1.91-cp36-cp36m-manylinux1_x86_64.whl (1.1MB)\n",
      "\u001b[K     |████████████████████████████████| 1.1MB 50.8MB/s \n",
      "\u001b[?25hRequirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.6/dist-packages (from transformers) (4.41.1)\n",
      "Requirement already satisfied: numpy in /usr/local/lib/python3.6/dist-packages (from transformers) (1.18.5)\n",
      "Requirement already satisfied: filelock in /usr/local/lib/python3.6/dist-packages (from transformers) (3.0.12)\n",
      "Requirement already satisfied: packaging in /usr/local/lib/python3.6/dist-packages (from transformers) (20.4)\n",
      "Requirement already satisfied: requests in /usr/local/lib/python3.6/dist-packages (from transformers) (2.23.0)\n",
      "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.6/dist-packages (from transformers) (2019.12.20)\n",
      "Requirement already satisfied: dataclasses; python_version < \"3.7\" in /usr/local/lib/python3.6/dist-packages (from transformers) (0.7)\n",
      "Requirement already satisfied: six in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers) (1.15.0)\n",
      "Requirement already satisfied: click in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers) (7.1.2)\n",
      "Requirement already satisfied: joblib in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers) (0.16.0)\n",
      "Requirement already satisfied: pyparsing>=2.0.2 in /usr/local/lib/python3.6/dist-packages (from packaging->transformers) (2.4.7)\n",
      "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (3.0.4)\n",
      "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (2.10)\n",
      "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (1.24.3)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (2020.6.20)\n",
      "Building wheels for collected packages: sacremoses\n",
      "  Building wheel for sacremoses (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
      "  Created wheel for sacremoses: filename=sacremoses-0.0.43-cp36-none-any.whl size=893257 sha256=e5d3cbc059a06c56b6b1f54cb870993c798e6986da6b5de019d00f85063b15b0\n",
      "  Stored in directory: /root/.cache/pip/wheels/29/3c/fd/7ce5c3f0666dab31a50123635e6fb5e19ceb42ce38d4e58f45\n",
      "Successfully built sacremoses\n",
      "Installing collected packages: sacremoses, tokenizers, sentencepiece, transformers\n",
      "Successfully installed sacremoses-0.0.43 sentencepiece-0.1.91 tokenizers-0.8.1rc2 transformers-3.3.1\n"
     ]
    }
   ],
   "source": [
    "!pip install transformers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "id": "RPz2ld5rEYd_"
   },
   "outputs": [],
   "source": [
    "import numpy as np # linear algebra\n",
    "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n",
    "from pathlib import Path \n",
    "\n",
    "import os\n",
    "\n",
    "import torch\n",
    "import torch.optim as optim\n",
    "\n",
    "import random \n",
    "\n",
    "# fastai\n",
    "from fastai import *\n",
    "from fastai.text import *\n",
    "from fastai.callbacks import *\n",
    "\n",
    "# transformers\n",
    "from transformers import PreTrainedModel, PreTrainedTokenizer, PretrainedConfig\n",
    "\n",
    "from transformers import BertForSequenceClassification, BertTokenizer, BertConfig\n",
    "from transformers import RobertaForSequenceClassification, RobertaTokenizer, RobertaConfig\n",
    "from transformers import XLNetForSequenceClassification, XLNetTokenizer, XLNetConfig\n",
    "from transformers import XLMForSequenceClassification, XLMTokenizer, XLMConfig\n",
    "from transformers import DistilBertForSequenceClassification, DistilBertTokenizer, DistilBertConfig"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "id": "zzeL6cLgzgJd",
    "outputId": "fdb8639f-2c8d-4ee3-8849-3d42cfb429a3"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mounted at /content/drive\n"
     ]
    }
   ],
   "source": [
    "from google.colab import drive\n",
    "drive.mount('/content/drive')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 195
    },
    "id": "5KtGKpfOE7u3",
    "outputId": "5f49f7a3-e38f-452a-f9b7-41947cb88b53"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>title</th>\n",
       "      <th>body</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>y-zoom piano roll</td>\n",
       "      <td>a y-zoom on the piano roll would be useful.</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>buggy behavior in selection</td>\n",
       "      <td>! screenshot from 2016-02-23 21 27 40  https:/...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>auto update feature</td>\n",
       "      <td>hi,\\r \\r great job so far, @saenzramiro ! : \\r...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>filter out noisy endpoints in logs</td>\n",
       "      <td>i think we should stop logging requests to:\\r ...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>enable pid on / pid off alarm actions for ardu...</td>\n",
       "      <td>expected behavior\\r alarm actions pid on and p...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                               title  ... label\n",
       "0                                  y-zoom piano roll  ...     1\n",
       "1                        buggy behavior in selection  ...     0\n",
       "2                                auto update feature  ...     1\n",
       "3                 filter out noisy endpoints in logs  ...     1\n",
       "4  enable pid on / pid off alarm actions for ardu...  ...     0\n",
       "\n",
       "[5 rows x 3 columns]"
      ]
     },
     "execution_count": 4,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_df = pd.read_json(\"/content/drive/My Drive/embold_train.json\").reset_index(drop=True)\n",
    "train_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 195
    },
    "id": "cezM6uwBGWdS",
    "outputId": "2f16b6af-725a-4d16-a263-68af74c0ee4d"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>title</th>\n",
       "      <th>body</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>config question  path-specific environment var...</td>\n",
       "      <td>issue description or question\\r \\r hey @artemg...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>crash indien vol</td>\n",
       "      <td>de simulator crasht als hij vol zit</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>unable to mine rocks</td>\n",
       "      <td>sarkasmo starting today, when i hit enter  act...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>not all whitelists are processed</td>\n",
       "      <td>create following rules... order of creation is...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>add ctx menu for idafree 70 and idafree 5</td>\n",
       "      <td>associated with .dll, .dll_, .exe, .exe_, .sc,...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                               title                                               body\n",
       "0  config question  path-specific environment var...  issue description or question\\r \\r hey @artemg...\n",
       "1                                   crash indien vol                de simulator crasht als hij vol zit\n",
       "2                               unable to mine rocks  sarkasmo starting today, when i hit enter  act...\n",
       "3                   not all whitelists are processed  create following rules... order of creation is...\n",
       "4          add ctx menu for idafree 70 and idafree 5  associated with .dll, .dll_, .exe, .exe_, .sc,..."
      ]
     },
     "execution_count": 5,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_df = pd.read_json(\"/content/drive/My Drive/embold_test.json\").reset_index(drop=True)\n",
    "test_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 195
    },
    "id": "YmcWYJBOGedE",
    "outputId": "caae3a51-14f8-4fb1-d047-e30dcb284b1f"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>title</th>\n",
       "      <th>body</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>use a 8bit typeface</td>\n",
       "      <td>since this is meant to emulate some old arcade...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>implement wireless m-bus binding</td>\n",
       "      <td>_from  chris.pa...@googlemail.com  https://cod...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>add multilang support for timeago.js</td>\n",
       "      <td>currently it is only  en . \\r required to add ...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>scaleway - seg-fault on shutdown</td>\n",
       "      <td>tbr  irc  creates a new scaleway instance with...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>sistema de pintura: no se guardar los nuevos p...</td>\n",
       "      <td>este sp ya estaba asignado a un carro y se enc...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                               title  ... label\n",
       "0                                use a 8bit typeface  ...     1\n",
       "1                   implement wireless m-bus binding  ...     1\n",
       "2               add multilang support for timeago.js  ...     1\n",
       "3                   scaleway - seg-fault on shutdown  ...     0\n",
       "4  sistema de pintura: no se guardar los nuevos p...  ...     0\n",
       "\n",
       "[5 rows x 3 columns]"
      ]
     },
     "execution_count": 6,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_ex_df = pd.read_json(\"/content/drive/My Drive/embold_train_extra.json\")\n",
    "train_ex_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 195
    },
    "id": "NTwS6tl0GlLh",
    "outputId": "dd18b34d-8906-49c0-dc09-973fd91f093e"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>title</th>\n",
       "      <th>body</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>y-zoom piano roll</td>\n",
       "      <td>y-zoom piano roll a y-zoom on the piano roll w...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>buggy behavior in selection</td>\n",
       "      <td>buggy behavior in selection ! screenshot from ...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>auto update feature</td>\n",
       "      <td>auto update feature hi,\\r \\r great job so far,...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>filter out noisy endpoints in logs</td>\n",
       "      <td>filter out noisy endpoints in logs i think we ...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>enable pid on / pid off alarm actions for ardu...</td>\n",
       "      <td>enable pid on / pid off alarm actions for ardu...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                               title  ... label\n",
       "0                                  y-zoom piano roll  ...     1\n",
       "1                        buggy behavior in selection  ...     0\n",
       "2                                auto update feature  ...     1\n",
       "3                 filter out noisy endpoints in logs  ...     1\n",
       "4  enable pid on / pid off alarm actions for ardu...  ...     0\n",
       "\n",
       "[5 rows x 3 columns]"
      ]
     },
     "execution_count": 7,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_df['body']=train_df['title']+\" \"+train_df['body']\n",
    "train_ex_df['body']=train_ex_df['title']+\" \"+train_ex_df['body']\n",
    "\n",
    "train_df1=pd.concat([train_df,train_ex_df],axis=0)\n",
    "train_df1.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 50
    },
    "id": "QmmRzcaFGwPM",
    "outputId": "80330538-c9f5-40a8-c4cd-fe0d9abc868e"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "fastai version : 1.0.61\n",
      "transformers version : 3.3.1\n"
     ]
    }
   ],
   "source": [
    "import fastai\n",
    "import transformers\n",
    "print('fastai version :', fastai.__version__)\n",
    "print('transformers version :', transformers.__version__)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "id": "HlheMtIxG4Ly"
   },
   "outputs": [],
   "source": [
    "MODEL_CLASSES = {\n",
    "    'bert': (BertForSequenceClassification, BertTokenizer, BertConfig),\n",
    "    'xlnet': (XLNetForSequenceClassification, XLNetTokenizer, XLNetConfig),\n",
    "    'xlm': (XLMForSequenceClassification, XLMTokenizer, XLMConfig),\n",
    "    'roberta': (RobertaForSequenceClassification, RobertaTokenizer, RobertaConfig),\n",
    "    'distilbert': (DistilBertForSequenceClassification, DistilBertTokenizer, DistilBertConfig)\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "id": "8I02YRgkHD-5"
   },
   "outputs": [],
   "source": [
    "# Parameters\n",
    "seed = 202020\n",
    "use_fp16 = False\n",
    "bs = 16\n",
    "\n",
    "model_type = 'roberta'\n",
    "pretrained_model_name = 'roberta-base'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "id": "qBBNohtQHOXq"
   },
   "outputs": [],
   "source": [
    "model_class, tokenizer_class, config_class = MODEL_CLASSES[model_type]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "id": "KBv1XPknHszE"
   },
   "outputs": [],
   "source": [
    "def seed_all(seed_value):\n",
    "    random.seed(seed_value) # Python\n",
    "    np.random.seed(seed_value) # cpu vars\n",
    "    torch.manual_seed(seed_value) # cpu  vars\n",
    "    \n",
    "    if torch.cuda.is_available(): \n",
    "        torch.cuda.manual_seed(seed_value)\n",
    "        torch.cuda.manual_seed_all(seed_value) # gpu vars\n",
    "        torch.backends.cudnn.deterministic = True  #needed\n",
    "        torch.backends.cudnn.benchmark = False\n",
    "seed_all(seed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "id": "jJo0rilIHswU"
   },
   "outputs": [],
   "source": [
    "class TransformersBaseTokenizer(BaseTokenizer):\n",
    "    \"\"\"Wrapper around PreTrainedTokenizer to be compatible with fast.ai\"\"\"\n",
    "    def __init__(self, pretrained_tokenizer: PreTrainedTokenizer, model_type = 'bert', **kwargs):\n",
    "        self._pretrained_tokenizer = pretrained_tokenizer\n",
    "        self.max_seq_len = pretrained_tokenizer.max_len\n",
    "        self.model_type = model_type\n",
    "\n",
    "    def __call__(self, *args, **kwargs): \n",
    "        return self\n",
    "\n",
    "    def tokenizer(self, t:str) -> List[str]:\n",
    "        \"\"\"Limits the maximum sequence length and add the spesial tokens\"\"\"\n",
    "        CLS = self._pretrained_tokenizer.cls_token\n",
    "        SEP = self._pretrained_tokenizer.sep_token\n",
    "        if self.model_type in ['roberta']:\n",
    "            tokens = self._pretrained_tokenizer.tokenize(t, add_prefix_space=True)[:self.max_seq_len - 2]\n",
    "            tokens = [CLS] + tokens + [SEP]\n",
    "        else:\n",
    "            tokens = self._pretrained_tokenizer.tokenize(t)[:self.max_seq_len - 2]\n",
    "            if self.model_type in ['xlnet']:\n",
    "                tokens = tokens + [SEP] +  [CLS]\n",
    "            else:\n",
    "                tokens = [CLS] + tokens + [SEP]\n",
    "        return tokens"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 168,
     "referenced_widgets": [
      "956c220b32b540f282d198583b19cd83",
      "476732bced504081baa9d838b4e2ce1e",
      "b5dbeb32d0824478a0e0bec4ee5e4834",
      "130f67d003784be4b1178e2847f3527e",
      "259d27bec617486c9c146ea77f99b488",
      "7c77338ad5904b2da07e9917cf8bf2a6",
      "bdb04ac4a9394c72a7f74c64e8dfe44c",
      "717db8fb535c40319b1c1f0d348c6f0d",
      "6d87fa5b6f4046b4892dd2934d25b87f",
      "4ad144ee7d2545efbc2b33b5c48acdc4",
      "5f35e41ac97a4f85a037d970351715a6",
      "8912c9db83b5450aa9dec6b2571dc4a2",
      "4a5b322be1bf4da29989a67346fec70f",
      "f2ef9acb3545464caefb040ced4e70c0",
      "ea9da948ded74743bd1a3d47a8e9d753",
      "2d63cfdd601e4170afebd9ccede64520"
     ]
    },
    "id": "M8IRcytHIAP_",
    "outputId": "72b0ae6c-1eee-4676-83dd-a6fce9c6b9fc"
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "956c220b32b540f282d198583b19cd83",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=898823.0, style=ProgressStyle(descripti…"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6d87fa5b6f4046b4892dd2934d25b87f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=456318.0, style=ProgressStyle(descripti…"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/dist-packages/transformers/tokenization_utils_base.py:1324: FutureWarning: The `max_len` attribute has been deprecated and will be removed in a future version, use `model_max_length` instead.\n",
      "  FutureWarning,\n"
     ]
    }
   ],
   "source": [
    "transformer_tokenizer = tokenizer_class.from_pretrained(pretrained_model_name)\n",
    "transformer_base_tokenizer = TransformersBaseTokenizer(pretrained_tokenizer = transformer_tokenizer, model_type = model_type)\n",
    "fastai_tokenizer = Tokenizer(tok_func = transformer_base_tokenizer, pre_rules=[], post_rules=[])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "id": "ArBvUsphIBxI"
   },
   "outputs": [],
   "source": [
    "class TransformersVocab(Vocab):\n",
    "    def __init__(self, tokenizer: PreTrainedTokenizer):\n",
    "        super(TransformersVocab, self).__init__(itos = [])\n",
    "        self.tokenizer = tokenizer\n",
    "    \n",
    "    def numericalize(self, t:Collection[str]) -> List[int]:\n",
    "        \"Convert a list of tokens `t` to their ids.\"\n",
    "        return self.tokenizer.convert_tokens_to_ids(t)\n",
    "        #return self.tokenizer.encode(t)\n",
    "\n",
    "    def textify(self, nums:Collection[int], sep=' ') -> List[str]:\n",
    "        \"Convert a list of `nums` to their tokens.\"\n",
    "        nums = np.array(nums).tolist()\n",
    "        return sep.join(self.tokenizer.convert_ids_to_tokens(nums)) if sep is not None else self.tokenizer.convert_ids_to_tokens(nums)\n",
    "    \n",
    "    def __getstate__(self):\n",
    "        return {'itos':self.itos, 'tokenizer':self.tokenizer}\n",
    "\n",
    "    def __setstate__(self, state:dict):\n",
    "        self.itos = state['itos']\n",
    "        self.tokenizer = state['tokenizer']\n",
    "        self.stoi = collections.defaultdict(int,{v:k for k,v in enumerate(self.itos)})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "id": "WXNsRL7kIB3l"
   },
   "outputs": [],
   "source": [
    "transformer_vocab =  TransformersVocab(tokenizer = transformer_tokenizer)\n",
    "numericalize_processor = NumericalizeProcessor(vocab=transformer_vocab)\n",
    "\n",
    "tokenize_processor = TokenizeProcessor(tokenizer=fastai_tokenizer, include_bos=False, include_eos=False)\n",
    "\n",
    "transformer_processor = [tokenize_processor, numericalize_processor]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "id": "_Eya6MkFIBub"
   },
   "outputs": [],
   "source": [
    "pad_first = bool(model_type in ['xlnet'])\n",
    "pad_idx = transformer_tokenizer.pad_token_id"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 67
    },
    "id": "lLfcUPZaIcfR",
    "outputId": "adbcfe20-cd91-4ae5-94c4-261974afb824"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['Sal', 'ut', 'Ġc', 'Ġest', 'Ġmo', 'i', ',', 'ĠHello', 'Ġit', 'Ġs', 'Ġme']\n",
      "[18111, 1182, 740, 3304, 7458, 118, 6, 20920, 24, 579, 162]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "['Sal', 'ut', 'Ġc', 'Ġest', 'Ġmo', 'i', ',', 'ĠHello', 'Ġit', 'Ġs', 'Ġme']"
      ]
     },
     "execution_count": 18,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokens = transformer_tokenizer.tokenize('Salut c est moi, Hello it s me')\n",
    "print(tokens)\n",
    "ids = transformer_tokenizer.convert_tokens_to_ids(tokens)\n",
    "print(ids)\n",
    "transformer_tokenizer.convert_ids_to_tokens(ids)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 17
    },
    "id": "kYhULwS4IcnB",
    "outputId": "fee5a1f8-2c7c-479f-b95d-1ee25a97d106"
   },
   "outputs": [
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "databunch = (TextList.from_df(train_df1, cols='body', processor=transformer_processor)\n",
    "             .split_by_rand_pct(0.05,seed=seed)\n",
    "             .label_from_df(cols= 'label')\n",
    "             .add_test(test_df)\n",
    "             .databunch(bs=bs, pad_first=pad_first, pad_idx=pad_idx))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "id": "I136s9ZzIunS"
   },
   "outputs": [],
   "source": [
    "# defining our model architecture \n",
    "class CustomTransformerModel(nn.Module):\n",
    "    def __init__(self, transformer_model: PreTrainedModel):\n",
    "        super(CustomTransformerModel,self).__init__()\n",
    "        self.transformer = transformer_model\n",
    "        \n",
    "    def forward(self, input_ids, attention_mask=None):\n",
    "        \n",
    "        # attention_mask\n",
    "        # Mask to avoid performing attention on padding token indices.\n",
    "        # Mask values selected in ``[0, 1]``:\n",
    "        # ``1`` for tokens that are NOT MASKED, ``0`` for MASKED tokens.\n",
    "        attention_mask = (input_ids!=pad_idx).type(input_ids.type()) \n",
    "        \n",
    "        logits = self.transformer(input_ids,\n",
    "                                  attention_mask = attention_mask)[0]   \n",
    "        return logits\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 620,
     "referenced_widgets": [
      "9ab422d8880a48d3a26bfeca87c30f17",
      "bde57c476b1a40549280fd30b9c3580f",
      "0dd0eb1c74eb447fbc6595dc7064405c",
      "f201fc153dc64bed99019c35c751a6ab",
      "cb4af4030f764ecdbd007d7555384294",
      "1e243edc2d6a483c9a1307f43602bb9a",
      "ed347becdda74efdb8290a2703867b17",
      "8b7caa3fe3e54a60811d8d1091ef48c1"
     ]
    },
    "id": "4K3HVNa4I7GK",
    "outputId": "02d70b1d-3541-44c7-d5f4-cb58cd78d159"
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9ab422d8880a48d3a26bfeca87c30f17",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=481.0, style=ProgressStyle(description_…"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "RobertaConfig {\n",
      "  \"architectures\": [\n",
      "    \"RobertaForMaskedLM\"\n",
      "  ],\n",
      "  \"attention_probs_dropout_prob\": 0.1,\n",
      "  \"bos_token_id\": 0,\n",
      "  \"eos_token_id\": 2,\n",
      "  \"gradient_checkpointing\": false,\n",
      "  \"hidden_act\": \"gelu\",\n",
      "  \"hidden_dropout_prob\": 0.1,\n",
      "  \"hidden_size\": 768,\n",
      "  \"id2label\": {\n",
      "    \"0\": \"LABEL_0\",\n",
      "    \"1\": \"LABEL_1\",\n",
      "    \"2\": \"LABEL_2\"\n",
      "  },\n",
      "  \"initializer_range\": 0.02,\n",
      "  \"intermediate_size\": 3072,\n",
      "  \"label2id\": {\n",
      "    \"LABEL_0\": 0,\n",
      "    \"LABEL_1\": 1,\n",
      "    \"LABEL_2\": 2\n",
      "  },\n",
      "  \"layer_norm_eps\": 1e-05,\n",
      "  \"max_position_embeddings\": 514,\n",
      "  \"model_type\": \"roberta\",\n",
      "  \"num_attention_heads\": 12,\n",
      "  \"num_hidden_layers\": 12,\n",
      "  \"pad_token_id\": 1,\n",
      "  \"type_vocab_size\": 1,\n",
      "  \"vocab_size\": 50265\n",
      "}\n",
      "\n"
     ]
    }
   ],
   "source": [
    "config = config_class.from_pretrained(pretrained_model_name)\n",
    "config.num_labels = 3\n",
    "config.use_bfloat16 = use_fp16\n",
    "print(config)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 170,
     "referenced_widgets": [
      "1fe38b4c12554aaea903d0116b5483a9",
      "c2ee60b580c349cc8bb4612e159e4c34",
      "8715b7c80dd846d499d6f13f41c5b9ff",
      "481df8579f9e47f28d8bcfa0157bf242",
      "84da667b57ee4dbd8cc29888a4ddcd84",
      "e322f2029bc94a90ac48089627a52848",
      "a0d5af49e8534efdaf351c57f3a451ce",
      "714f8746bc2b4ca7acb02f1e46ae2649"
     ]
    },
    "id": "kdRIljYhI7Mi",
    "outputId": "9d3e5bc6-25eb-481b-edfd-d17b6dd3df26"
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1fe38b4c12554aaea903d0116b5483a9",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=501200538.0, style=ProgressStyle(descri…"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of the model checkpoint at roberta-base were not used when initializing RobertaForSequenceClassification: ['lm_head.bias', 'lm_head.dense.weight', 'lm_head.dense.bias', 'lm_head.layer_norm.weight', 'lm_head.layer_norm.bias', 'lm_head.decoder.weight', 'roberta.pooler.dense.weight', 'roberta.pooler.dense.bias']\n",
      "- This IS expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPretraining model).\n",
      "- This IS NOT expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
      "Some weights of RobertaForSequenceClassification were not initialized from the model checkpoint at roberta-base and are newly initialized: ['classifier.dense.weight', 'classifier.dense.bias', 'classifier.out_proj.weight', 'classifier.out_proj.bias']\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
     ]
    }
   ],
   "source": [
    "transformer_model = model_class.from_pretrained(pretrained_model_name, config = config)\n",
    "\n",
    "custom_transformer_model = CustomTransformerModel(transformer_model = transformer_model)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "id": "loo-WYhMI7DY"
   },
   "outputs": [],
   "source": [
    "from fastai.callbacks import *\n",
    "from transformers import AdamW\n",
    "from functools import partial\n",
    "\n",
    "CustomAdamW = partial(AdamW, correct_bias=False)\n",
    "\n",
    "learner = Learner(databunch, \n",
    "                  custom_transformer_model, \n",
    "                  opt_func = CustomAdamW, \n",
    "                  metrics=[accuracy, error_rate])\n",
    "\n",
    "# Show graph of learner stats and metrics after each epoch.\n",
    "learner.callbacks.append(ShowGraph(learner))\n",
    "\n",
    "# Put learn in FP16 precision mode. --> Seems to not working\n",
    "if use_fp16: learner = learner.to_fp16()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "id": "mGzwjMMiJIFf"
   },
   "outputs": [],
   "source": [
    "# For roberta-base\n",
    "list_layers = [learner.model.transformer.roberta.embeddings,\n",
    "              learner.model.transformer.roberta.encoder.layer[0],\n",
    "              learner.model.transformer.roberta.encoder.layer[1],\n",
    "              learner.model.transformer.roberta.encoder.layer[2],\n",
    "              learner.model.transformer.roberta.encoder.layer[3],\n",
    "              learner.model.transformer.roberta.encoder.layer[4],\n",
    "              learner.model.transformer.roberta.encoder.layer[5],\n",
    "              learner.model.transformer.roberta.encoder.layer[6],\n",
    "              learner.model.transformer.roberta.encoder.layer[7],\n",
    "              learner.model.transformer.roberta.encoder.layer[8],\n",
    "              learner.model.transformer.roberta.encoder.layer[9],\n",
    "              learner.model.transformer.roberta.encoder.layer[10],\n",
    "              learner.model.transformer.roberta.encoder.layer[11],\n",
    "              learner.model.transformer.roberta.pooler]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000
    },
    "id": "lp4k1KT4oioU",
    "outputId": "39e6c2cb-55a4-4796-f9fd-7ccec89fbb56"
   },
   "outputs": [
    {
     "data": {
      "application/vnd.google.colaboratory.intrinsic+json": {
       "type": "string"
      },
      "text/plain": [
       "CustomTransformerModel\n",
       "======================================================================\n",
       "Layer (type)         Output Shape         Param #    Trainable \n",
       "======================================================================\n",
       "Embedding            [512, 768]           38,603,520 True      \n",
       "______________________________________________________________________\n",
       "Embedding            [512, 768]           394,752    True      \n",
       "______________________________________________________________________\n",
       "Embedding            [512, 768]           768        True      \n",
       "______________________________________________________________________\n",
       "LayerNorm            [512, 768]           1,536      True      \n",
       "______________________________________________________________________\n",
       "Dropout              [512, 768]           0          False     \n",
       "______________________________________________________________________\n",
       "Linear               [512, 768]           590,592    True      \n",
       "______________________________________________________________________\n",
       "Linear               [512, 768]           590,592    True      \n",
       "______________________________________________________________________\n",
       "Linear               [512, 768]           590,592    True      \n",
       "______________________________________________________________________\n",
       "Dropout              [12, 512, 512]       0          False     \n",
       "______________________________________________________________________\n",
       "Linear               [512, 768]           590,592    True      \n",
       "______________________________________________________________________\n",
       "LayerNorm            [512, 768]           1,536      True      \n",
       "______________________________________________________________________\n",
       "Dropout              [512, 768]           0          False     \n",
       "______________________________________________________________________\n",
       "Linear               [512, 3072]          2,362,368  True      \n",
       "______________________________________________________________________\n",
       "Linear               [512, 768]           2,360,064  True      \n",
       "______________________________________________________________________\n",
       "LayerNorm            [512, 768]           1,536      True      \n",
       "______________________________________________________________________\n",
       "Dropout              [512, 768]           0          False     \n",
       "______________________________________________________________________\n",
       "Linear               [512, 768]           590,592    True      \n",
       "______________________________________________________________________\n",
       "Linear               [512, 768]           590,592    True      \n",
       "______________________________________________________________________\n",
       "Linear               [512, 768]           590,592    True      \n",
       "______________________________________________________________________\n",
       "Dropout              [12, 512, 512]       0          False     \n",
       "______________________________________________________________________\n",
       "Linear               [512, 768]           590,592    True      \n",
       "______________________________________________________________________\n",
       "LayerNorm            [512, 768]           1,536      True      \n",
       "______________________________________________________________________\n",
       "Dropout              [512, 768]           0          False     \n",
       "______________________________________________________________________\n",
       "Linear               [512, 3072]          2,362,368  True      \n",
       "______________________________________________________________________\n",
       "Linear               [512, 768]           2,360,064  True      \n",
       "______________________________________________________________________\n",
       "LayerNorm            [512, 768]           1,536      True      \n",
       "______________________________________________________________________\n",
       "Dropout              [512, 768]           0          False     \n",
       "______________________________________________________________________\n",
       "Linear               [512, 768]           590,592    True      \n",
       "______________________________________________________________________\n",
       "Linear               [512, 768]           590,592    True      \n",
       "______________________________________________________________________\n",
       "Linear               [512, 768]           590,592    True      \n",
       "______________________________________________________________________\n",
       "Dropout              [12, 512, 512]       0          False     \n",
       "______________________________________________________________________\n",
       "Linear               [512, 768]           590,592    True      \n",
       "______________________________________________________________________\n",
       "LayerNorm            [512, 768]           1,536      True      \n",
       "______________________________________________________________________\n",
       "Dropout              [512, 768]           0          False     \n",
       "______________________________________________________________________\n",
       "Linear               [512, 3072]          2,362,368  True      \n",
       "______________________________________________________________________\n",
       "Linear               [512, 768]           2,360,064  True      \n",
       "______________________________________________________________________\n",
       "LayerNorm            [512, 768]           1,536      True      \n",
       "______________________________________________________________________\n",
       "Dropout              [512, 768]           0          False     \n",
       "______________________________________________________________________\n",
       "Linear               [512, 768]           590,592    True      \n",
       "______________________________________________________________________\n",
       "Linear               [512, 768]           590,592    True      \n",
       "______________________________________________________________________\n",
       "Linear               [512, 768]           590,592    True      \n",
       "______________________________________________________________________\n",
       "Dropout              [12, 512, 512]       0          False     \n",
       "______________________________________________________________________\n",
       "Linear               [512, 768]           590,592    True      \n",
       "______________________________________________________________________\n",
       "LayerNorm            [512, 768]           1,536      True      \n",
       "______________________________________________________________________\n",
       "Dropout              [512, 768]           0          False     \n",
       "______________________________________________________________________\n",
       "Linear               [512, 3072]          2,362,368  True      \n",
       "______________________________________________________________________\n",
       "Linear               [512, 768]           2,360,064  True      \n",
       "______________________________________________________________________\n",
       "LayerNorm            [512, 768]           1,536      True      \n",
       "______________________________________________________________________\n",
       "Dropout              [512, 768]           0          False     \n",
       "______________________________________________________________________\n",
       "Linear               [512, 768]           590,592    True      \n",
       "______________________________________________________________________\n",
       "Linear               [512, 768]           590,592    True      \n",
       "______________________________________________________________________\n",
       "Linear               [512, 768]           590,592    True      \n",
       "______________________________________________________________________\n",
       "Dropout              [12, 512, 512]       0          False     \n",
       "______________________________________________________________________\n",
       "Linear               [512, 768]           590,592    True      \n",
       "______________________________________________________________________\n",
       "LayerNorm            [512, 768]           1,536      True      \n",
       "______________________________________________________________________\n",
       "Dropout              [512, 768]           0          False     \n",
       "______________________________________________________________________\n",
       "Linear               [512, 3072]          2,362,368  True      \n",
       "______________________________________________________________________\n",
       "Linear               [512, 768]           2,360,064  True      \n",
       "______________________________________________________________________\n",
       "LayerNorm            [512, 768]           1,536      True      \n",
       "______________________________________________________________________\n",
       "Dropout              [512, 768]           0          False     \n",
       "______________________________________________________________________\n",
       "Linear               [512, 768]           590,592    True      \n",
       "______________________________________________________________________\n",
       "Linear               [512, 768]           590,592    True      \n",
       "______________________________________________________________________\n",
       "Linear               [512, 768]           590,592    True      \n",
       "______________________________________________________________________\n",
       "Dropout              [12, 512, 512]       0          False     \n",
       "______________________________________________________________________\n",
       "Linear               [512, 768]           590,592    True      \n",
       "______________________________________________________________________\n",
       "LayerNorm            [512, 768]           1,536      True      \n",
       "______________________________________________________________________\n",
       "Dropout              [512, 768]           0          False     \n",
       "______________________________________________________________________\n",
       "Linear               [512, 3072]          2,362,368  True      \n",
       "______________________________________________________________________\n",
       "Linear               [512, 768]           2,360,064  True      \n",
       "______________________________________________________________________\n",
       "LayerNorm            [512, 768]           1,536      True      \n",
       "______________________________________________________________________\n",
       "Dropout              [512, 768]           0          False     \n",
       "______________________________________________________________________\n",
       "Linear               [512, 768]           590,592    True      \n",
       "______________________________________________________________________\n",
       "Linear               [512, 768]           590,592    True      \n",
       "______________________________________________________________________\n",
       "Linear               [512, 768]           590,592    True      \n",
       "______________________________________________________________________\n",
       "Dropout              [12, 512, 512]       0          False     \n",
       "______________________________________________________________________\n",
       "Linear               [512, 768]           590,592    True      \n",
       "______________________________________________________________________\n",
       "LayerNorm            [512, 768]           1,536      True      \n",
       "______________________________________________________________________\n",
       "Dropout              [512, 768]           0          False     \n",
       "______________________________________________________________________\n",
       "Linear               [512, 3072]          2,362,368  True      \n",
       "______________________________________________________________________\n",
       "Linear               [512, 768]           2,360,064  True      \n",
       "______________________________________________________________________\n",
       "LayerNorm            [512, 768]           1,536      True      \n",
       "______________________________________________________________________\n",
       "Dropout              [512, 768]           0          False     \n",
       "______________________________________________________________________\n",
       "Linear               [512, 768]           590,592    True      \n",
       "______________________________________________________________________\n",
       "Linear               [512, 768]           590,592    True      \n",
       "______________________________________________________________________\n",
       "Linear               [512, 768]           590,592    True      \n",
       "______________________________________________________________________\n",
       "Dropout              [12, 512, 512]       0          False     \n",
       "______________________________________________________________________\n",
       "Linear               [512, 768]           590,592    True      \n",
       "______________________________________________________________________\n",
       "LayerNorm            [512, 768]           1,536      True      \n",
       "______________________________________________________________________\n",
       "Dropout              [512, 768]           0          False     \n",
       "______________________________________________________________________\n",
       "Linear               [512, 3072]          2,362,368  True      \n",
       "______________________________________________________________________\n",
       "Linear               [512, 768]           2,360,064  True      \n",
       "______________________________________________________________________\n",
       "LayerNorm            [512, 768]           1,536      True      \n",
       "______________________________________________________________________\n",
       "Dropout              [512, 768]           0          False     \n",
       "______________________________________________________________________\n",
       "Linear               [512, 768]           590,592    True      \n",
       "______________________________________________________________________\n",
       "Linear               [512, 768]           590,592    True      \n",
       "______________________________________________________________________\n",
       "Linear               [512, 768]           590,592    True      \n",
       "______________________________________________________________________\n",
       "Dropout              [12, 512, 512]       0          False     \n",
       "______________________________________________________________________\n",
       "Linear               [512, 768]           590,592    True      \n",
       "______________________________________________________________________\n",
       "LayerNorm            [512, 768]           1,536      True      \n",
       "______________________________________________________________________\n",
       "Dropout              [512, 768]           0          False     \n",
       "______________________________________________________________________\n",
       "Linear               [512, 3072]          2,362,368  True      \n",
       "______________________________________________________________________\n",
       "Linear               [512, 768]           2,360,064  True      \n",
       "______________________________________________________________________\n",
       "LayerNorm            [512, 768]           1,536      True      \n",
       "______________________________________________________________________\n",
       "Dropout              [512, 768]           0          False     \n",
       "______________________________________________________________________\n",
       "Linear               [512, 768]           590,592    True      \n",
       "______________________________________________________________________\n",
       "Linear               [512, 768]           590,592    True      \n",
       "______________________________________________________________________\n",
       "Linear               [512, 768]           590,592    True      \n",
       "______________________________________________________________________\n",
       "Dropout              [12, 512, 512]       0          False     \n",
       "______________________________________________________________________\n",
       "Linear               [512, 768]           590,592    True      \n",
       "______________________________________________________________________\n",
       "LayerNorm            [512, 768]           1,536      True      \n",
       "______________________________________________________________________\n",
       "Dropout              [512, 768]           0          False     \n",
       "______________________________________________________________________\n",
       "Linear               [512, 3072]          2,362,368  True      \n",
       "______________________________________________________________________\n",
       "Linear               [512, 768]           2,360,064  True      \n",
       "______________________________________________________________________\n",
       "LayerNorm            [512, 768]           1,536      True      \n",
       "______________________________________________________________________\n",
       "Dropout              [512, 768]           0          False     \n",
       "______________________________________________________________________\n",
       "Linear               [512, 768]           590,592    True      \n",
       "______________________________________________________________________\n",
       "Linear               [512, 768]           590,592    True      \n",
       "______________________________________________________________________\n",
       "Linear               [512, 768]           590,592    True      \n",
       "______________________________________________________________________\n",
       "Dropout              [12, 512, 512]       0          False     \n",
       "______________________________________________________________________\n",
       "Linear               [512, 768]           590,592    True      \n",
       "______________________________________________________________________\n",
       "LayerNorm            [512, 768]           1,536      True      \n",
       "______________________________________________________________________\n",
       "Dropout              [512, 768]           0          False     \n",
       "______________________________________________________________________\n",
       "Linear               [512, 3072]          2,362,368  True      \n",
       "______________________________________________________________________\n",
       "Linear               [512, 768]           2,360,064  True      \n",
       "______________________________________________________________________\n",
       "LayerNorm            [512, 768]           1,536      True      \n",
       "______________________________________________________________________\n",
       "Dropout              [512, 768]           0          False     \n",
       "______________________________________________________________________\n",
       "Linear               [512, 768]           590,592    True      \n",
       "______________________________________________________________________\n",
       "Linear               [512, 768]           590,592    True      \n",
       "______________________________________________________________________\n",
       "Linear               [512, 768]           590,592    True      \n",
       "______________________________________________________________________\n",
       "Dropout              [12, 512, 512]       0          False     \n",
       "______________________________________________________________________\n",
       "Linear               [512, 768]           590,592    True      \n",
       "______________________________________________________________________\n",
       "LayerNorm            [512, 768]           1,536      True      \n",
       "______________________________________________________________________\n",
       "Dropout              [512, 768]           0          False     \n",
       "______________________________________________________________________\n",
       "Linear               [512, 3072]          2,362,368  True      \n",
       "______________________________________________________________________\n",
       "Linear               [512, 768]           2,360,064  True      \n",
       "______________________________________________________________________\n",
       "LayerNorm            [512, 768]           1,536      True      \n",
       "______________________________________________________________________\n",
       "Dropout              [512, 768]           0          False     \n",
       "______________________________________________________________________\n",
       "Linear               [768]                590,592    True      \n",
       "______________________________________________________________________\n",
       "Dropout              [768]                0          False     \n",
       "______________________________________________________________________\n",
       "Linear               [3]                  2,307      True      \n",
       "______________________________________________________________________\n",
       "\n",
       "Total params: 124,647,939\n",
       "Total trainable params: 124,647,939\n",
       "Total non-trainable params: 0\n",
       "Optimized with 'transformers.optimization.AdamW', correct_bias=False\n",
       "Using true weight decay as discussed in https://www.fast.ai/2018/07/02/adam-weight-decay/ \n",
       "Loss function : FlattenedLoss\n",
       "======================================================================\n",
       "Callbacks functions applied \n",
       "    ShowGraph"
      ]
     },
     "execution_count": 25,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "learner.save('untrain')\n",
    "seed_all(seed)\n",
    "learner.load('untrain')\n",
    "learner.freeze_to(-1)\n",
    "learner.summary()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 110
    },
    "id": "AWnUoRNeo2XE",
    "outputId": "17694bd9-ff41-4dfe-b4a7-89770d1c5769"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "        <style>\n",
       "            /* Turns off some styling */\n",
       "            progress {\n",
       "                /* gets rid of default border in Firefox and Opera. */\n",
       "                border: none;\n",
       "                /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
       "                background-size: auto;\n",
       "            }\n",
       "            .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
       "                background: #F44336;\n",
       "            }\n",
       "        </style>\n",
       "      <progress value='0' class='' max='1' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      0.00% [0/1 00:00<00:00]\n",
       "    </div>\n",
       "    \n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>error_rate</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table><p>\n",
       "\n",
       "    <div>\n",
       "        <style>\n",
       "            /* Turns off some styling */\n",
       "            progress {\n",
       "                /* gets rid of default border in Firefox and Opera. */\n",
       "                border: none;\n",
       "                /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
       "                background-size: auto;\n",
       "            }\n",
       "            .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
       "                background: #F44336;\n",
       "            }\n",
       "        </style>\n",
       "      <progress value='73' class='' max='25312' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      0.29% [73/25312 00:53<5:08:22 3.7590]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LR Finder is complete, type {learner_name}.recorder.plot() to see the graph.\n"
     ]
    }
   ],
   "source": [
    "learner.lr_find()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 313
    },
    "id": "4QPj02gHo7LT",
    "outputId": "2db6e00c-bbe8-4b49-ad26-08e970a21549"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Min numerical gradient: 1.10E-04\n",
      "Min loss divided by 10: 1.32E-05\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEGCAYAAABo25JHAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3de3hc9X3n8fd3pNH9bsuWsSwbHLDNxeZiLiUkMU1LMU1Dtk3apYG22aRut2ma7LLZPG2fDWmz+2y32SRNlhLiTVk2aQptEtomBFJCAiGEBWobX7BljMHYFpas64wuo8tI+u4fc2QLoyvSmYvm83qeeaSZc2bOd36W5zPn/H7nd8zdERGR/BXJdAEiIpJZCgIRkTynIBARyXMKAhGRPKcgEBHJc4WZLmC+li9f7uvWrct0GSIiOWX37t2d7l4/1bKcC4J169axa9euTJchIpJTzOz4dMt0aEhEJM8pCERE8pyCQEQkzykIRETynIJARCTPKQhERPKcgkBEJM8pCEREcsCXHn+Zn77cEcprKwhERLLc+LjzpR8d4blXu0N5fQWBiEiW6x8ZZdyhpiwayusrCEREslw8kQSgulRBICKSl+KDCgIRkbwWC/YIasqKQnl9BYGISJaLDY4A6iMQEclbMfURiIjkN/URiIjkufhgkpJohJJoQSivryAQEclyscRIaHsDoCAQEcl68cEkNaXhjBgCBYGISNaLJZJUhzRiCBQEIiJZL7VHoCAQEclb8cGk+ghERPJZLJEM7WQyCDEIzGyNmT1hZofM7KCZfXyKdT5oZvvN7ICZPWNmW8KqR0QkFw0lxxhMjoU2vQRAYWivDKPAne6+x8wqgd1m9kN3PzRpnWPAu9y9x8y2AzuBa0OsSUQkp/QGJ5NVhXhoKLQgcPdWoDX4vc/MmoHVwKFJ6zwz6SnPAo1h1SMikosmzirO+c5iM1sHXAE8N8NqHwYeneb5O8xsl5nt6ugI51JtIiLZKDYRBLnYRzDBzCqA7wCfcPfeada5kVQQfGqq5e6+0923uvvW+vr68IoVEckyYU84B+H2EWBmUVIh8E13f2iadTYDXwO2u3tXmPWIiOSas4eGcvDMYjMz4G+AZnf/wjTrNAEPAXe4+5GwahERyVWxROpaBGGeWRzmHsHbgTuAA2a2N3jsT4AmAHe/F/g0sAy4J5UbjLr71hBrEhHJKfHBJBGDyuLwPq7DHDX0NGCzrPMR4CNh1SAikuvig0mqSqNEIjN+nC6IziwWEclisUS48wyBgkBEJKvFBpNUh3hWMSgIRESyWjzki9KAgkBEJKuFPQU1KAhERLJabDDcmUdBQSAikrXGxz30axGAgkBEJGv1DY/iHu70EqAgEBHJWvHExIRzGjUkIpKXYoPB9BLaIxARyU+xRPhTUIOCQEQka6XjojSgIBARyVoTF6UJc+ZRUBCIiGSteEJ9BCIieS0+mKQ0WkBxYUGo21EQiIhkqVgi/LOKQUEgIpK1Ymk4qxgUBCIiWSsd00uAgkBEJGvFdWhIRCS/xQbDvxYBKAhERLJWqrM43HmGQEEgIpKVhpJjDI+O5/YegZmtMbMnzOyQmR00s49PsY6Z2ZfN7KiZ7TezK8OqR0Qkl0xML5GOICgM8bVHgTvdfY+ZVQK7zeyH7n5o0jrbgQuD27XAV4KfIiJ5LV0TzkGIewTu3urue4Lf+4BmYPU5q90KfN1TngVqzGxVWDWJiOSKsxPOLZE+AjNbB1wBPHfOotXAyUn3W3hzWGBmO8xsl5nt6ujoCKtMEZGsEQvmGcrpPYIJZlYBfAf4hLv3vpXXcPed7r7V3bfW19cvboEiIlkolsY+glCDwMyipELgm+7+0BSrvA6smXS/MXhMRCSv9aZpCmoId9SQAX8DNLv7F6ZZ7bvAbwWjh64D4u7eGlZNIiK5IpZIUhAxKovDHNOTEuYW3g7cARwws73BY38CNAG4+73AI8AtwFEgAXwoxHpERHJGbHCEqpJCUt+pwxVaELj708CM78DdHfhoWDWIiOSqdJ1VDDqzWEQkK6Vr5lFQEIiIZCUFgYhInkvX1clAQSAikpXig0lqtEcgIpKfxsad3qEk1eosFhHJT31DSdzTc1YxKAhERLLO2QnnFAQiInkpnVNQg4JARCTrpHPCOVAQiIhknXROQQ0KAhGRrHNm5tE0XJQGFAQiIllnoo9Ah4ZERPJUbDBJWVEBRYXp+YhWEIiIZJl0nlUMCgIRkawTSySpUhCIiOSv+OBI2kYMgYJARCTrxBJJatI0YggUBCIiWSc+mL4pqEFBICKSVdydWBovSgMKAhGRrDKUHGdkdJzqpbBHYGb3mVm7mb04zfJqM/ueme0zs4Nm9qGwahERyRVnZx5dGn0E9wM3z7D8o8Ahd98CbAM+b2bpe+ciIlkoNpiaZ2hJHBpy96eA7plWASrNzICKYN3RsOoREckF6Z6CGjLbR3A3sAk4BRwAPu7u41OtaGY7zGyXme3q6OhIZ40iImkVT/MU1JDZIPglYC9wHnA5cLeZVU21orvvdPet7r61vr4+nTWKiKRVPM0TzkFmg+BDwEOechQ4BmzMYD0iIhk30UeQL4eGTgDvBjCzlcAG4NUM1iMiknGxRJKCiFFRXJi2bYa2JTN7gNRooOVm1gLcBUQB3P1e4LPA/WZ2ADDgU+7eGVY9IiK5YGLm0dQ4mvQILQjc/bZZlp8Cbgpr+yIiuSjdZxWDziwWEckqr7T301BdktZtKghERLLEqx39HG7r4xc2rUzrdhUEIiJZ4tEX2wDYfllDWrerIBARyRKPHGjlyqYaVlWXpnW7CgIRkSxwvGuAg6d6ueWyVWnftoJARCQLTBwWuvnS9B4WAgWBiEhWePRAK1saq2msLUv7thUEIiIZdrI7wb6WONszcFgI5hgEZlZuZpHg94vM7L1mlt4zHkRElqgfBIeFbrk0i4MAeAooMbPVwGPAHaQuPCMiIgv0yIutXLq6iqZl6T8sBHMPAnP3BPCrwD3u/gHgkvDKEhHJD6dig7xwIsb2DO0NwDyCwMx+Dvgg8P3gsYJwShIRyR8Th4W2Z2C00IS5BsEngD8G/tHdD5rZBcAT4ZUlIpIfHjnQysaGSi6or8hYDXOafdTdfwL8BCDoNO509z8KszARkaWuLT7EruM93PmLF2W0jrmOGvo7M6sys3LgReCQmX0y3NJERJa2fzk4MbdQ5voHYO6Hhi52917gfcCjwPmkRg6JiMhb9MiBVi5aWcHbVmTusBDMPQiiwXkD7wO+6+5JwMMrS0RkaevoG+b517ozOlpowlyD4KvAa0A58JSZrQV6wypKRGSpe7z5NO7pn3J6KnPtLP4y8OVJDx03sxvDKUlEZOnb3xKjtizKhpWVmS5lzp3F1Wb2BTPbFdw+T2rvQERE3oLm1j42NlSl9SL105nroaH7gD7g14NbL/B/ZnqCmd1nZu1m9uIM62wzs71mdtDMfjLXokVEctnYuPNSWx8bV2V+bwDmeGgIWO/uvzbp/p+Z2d5ZnnM/cDfw9akWmlkNcA9ws7ufMLMVc6xFRCSnnehOMJgcY9OqqkyXAsx9j2DQzG6YuGNmbwcGZ3qCuz8FdM+wym8CD7n7iWD99jnWIiKS0w63psbabGrIjiCY6x7B7wNfN7Pq4H4P8NsL3PZFpIalPglUAl9y9yn3HkRElpLm1l4iBheuzOz5AxPmOmpoH7DFzKqC+71m9glg/wK3fRXwbqAU+H9m9qy7Hzl3RTPbAewAaGpqWsAmRUQyr7mtjwvqKyiJZsfcnfO6Qpm79wZnGAP8xwVuuwX4F3cfcPdOUtc82DLNdne6+1Z331pfX7/AzYqIZNbhtl42NmRHRzEs7FKVCx3z9M/ADWZWaGZlwLVA8wJfU0Qkq/UNJTnZPZg1HcUw9z6Cqcw4xYSZPQBsA5abWQtwFxAFcPd73b3ZzH5A6vDSOPA1d592qKmIyFLwUlsfAJuyZOgozBIEZtbH1B/4Ruq4/rTc/bbZNu7unwM+N9t6IiJLRXMQBBuzZMQQzBIE7p49kSUisgQ0t/ZSVVLIquqSTJdyxkL6CEREZJ4Ot/ayaVV2TC0xQUEgIpIm48HUEtnUUQwKAhGRtDnZk2BgZCyrho6CgkBEJG2aWydGDGmPQEQkLx1u68UMLsqCaxBMpiAQEUmT5tZezl9WTmlRdkwtMUFBICKSJoezsKMYFAQiImkxMDzK8a5E1nUUg4JARCQtDk+cUaw9AhGR/HS4LTVxs/YIRETy1OHWPiqLC2msnXGatoxQEIiIpEFzay8bV1Vm1dQSExQEIiIhc3cOt/Vl1YyjkykIRERC1tIzSP/waFYOHQUFgYhI6Jpbg47iLLoYzWQKAhGRkB1u68MMNmTZ1BITFAQiIiE73NbL2royyosXcnXg8CgIRERC1tyavR3FoCAQEQlVYmSU17oGsrZ/ABQEIiKhOnSqF/fsuwbBZKEFgZndZ2btZvbiLOtdbWajZvb+sGoREcmUnx3twgyuWVeX6VKmFeYewf3AzTOtYGYFwP8AHguxDhGRjHn6aAeXra6mtrwo06VMK7QgcPengO5ZVvsY8B2gPaw6REQypX94lBdOxHj725ZnupQZZayPwMxWA/8G+Moc1t1hZrvMbFdHR0f4xYmILILnj3UxOu68Q0Ewrb8CPuXu47Ot6O473X2ru2+tr69PQ2kiIgv305c7KS6McOXa2kyXMqNMnt2wFXgwmIlvOXCLmY26+z9lsCYRkUXzs6OdXHN+HSXR7LpG8bkyFgTufv7E72Z2P/CwQkBElor23iGOnO7n165szHQpswotCMzsAWAbsNzMWoC7gCiAu98b1nZFRLLB00c7AbK+oxhCDAJ3v20e6/5OWHWIiGTC00c7qSsv4uIsPpFsgs4sFhFZZO7Oz452cv36ZUQi2XdFsnMpCEREFtnR9n5O9w5zQw4cFgIFgYjIopvoH7jhQgWBiEhe+tnRTtYtK6OxtizTpcyJgkBEZBElx8Z59tXunBgtNEFBICKyiPadjNE/PMo7cuSwECgIREQW1dNHO4kY/NwFCgIRkbz09MudXNZYQ3VZNNOlzJmCQERkkfQNJXnhZIwb3rYs06XMi4JARGSRPH+sm7Fxz6mOYlAQiIgsmp++3ElJNMJVWT7t9LkUBCIii+Tpo51cc/4yiguze9rpcykIREQWwSsd/Rxt7+fGDbl38SwFgYjIIvj+/lbM4JbLVmW6lHlTEIiILIKH95/i6nV1rKwqyXQp86YgEBFZoCOn+zhyup/3bM69vQFQEIiILNjD+1uJGNx8aUOmS3lLFAQiIgvg7jy8/xTXnr+MFZW5d1gIFAQiIgtyuK2PVzsGeM+W3DwsBAoCEZEFeXj/KQoixs2X5OZhIQgxCMzsPjNrN7MXp1n+QTPbb2YHzOwZM9sSVi0iImFwd76/v5Xr1y9jWUVxpst5y8LcI7gfuHmG5ceAd7n7ZcBngZ0h1iIisugOnurlta5Ezo4WmlAY1gu7+1Nmtm6G5c9Muvss0BhWLSIiYXh4fyuFEeOXcviwEGRPH8GHgUenW2hmO8xsl5nt6ujoSGNZIiJTmxgtdMOFy6kpK8p0OQuS8SAwsxtJBcGnplvH3Xe6+1Z331pfn3vzeIjI0rOvJU5LzyC/nINTSpwrtENDc2Fmm4GvAdvdvSuTtYiIzMf3958iWmDclOOHhSCDQWBmTcBDwB3ufiRTdYiIABxt72fcnZrSKNVl0Rmnkh4fT40WeueF9VSX5s4lKacTWhCY2QPANmC5mbUAdwFRAHe/F/g0sAy4x8wARt19a1j1iIhMZWzc+e+PNPO1p4+94fHSaAE1ZVGWVRSxYWUVm1ZVcvGqKjatquLVzgFOxYf45M0bMlT14gpz1NBtsyz/CPCRsLYvIjKb/uFRPv7AC/zocDu3X9fEdRcsI5ZIEh9MEkuMEEskaesd4qmXO/jOnpYzzyuNFlBUGOEXNq3MYPWLJ6N9BCIimdLSk+Aj/3cXL7f389lbL+GOn1s34/odfcM0t/aeuV18XhWVJbl/WAgUBCKSh3Yf7+H3vrGL4dFx7v/Q1bzjwtlHI9ZXFlNfWc87L1p6IxcVBCKSV7637xR3fmsfq6pLeHDHVt62ojLTJWWcgkBE8kZbfIg7v7WPzaur2flbW6krz+0TwRZLxk8oExFJl3uePMr4uPPF37hcITCJgkBE8sKp2CAPPn+SD2xtZE1dWabLySoKAhHJC1958hUc56M3vi3TpWQdBYGILHmnYoP8/b+e5ANb19BYq72BcykIRGTJ++snjmpvYAZ5P2rI3ekfHqWjbzh16x8+8ztAdWmU6tIoNWVRqkqj1JQWUVdeRG35zHORzLS9zv4RRsbGKSqIUFQYobgwQlFBhEjEFvvtieS9lp4E/7DrJL++dQ2ra0ozXU5WypsgOHgqzkN7Xqd7YISugRG6+ofP/D4yOv6m9QuDD+XRcZ/2NSuKC6krD4KhLEplSZTy4kIqSwopLyqkoqSQ0bFxWnoGOdmT4GR3gpaeQYan2B5AUUGE0qICyosKKCsupLyogPLiQkqjBRREjIKIEYkYBZb6vbKkkKa6Mprqyli7rJymujJKi+YfTiJL2T1PvoJh2huYQd4EQUvPIA88f4JlFUXUlRezsqqETauqWFZRxLLyIlZUlgRnDhZTX1FMTVnq1PHEyBixwSTxRJLY4AjxRJLuxAg9QYh0B7eO/mFe60rQNzRK/3CSoeTZD/vq0iiNtaVcuKKSGzesoLG2lNKiAkZGxxkObhO/D46MMjAyRmJklIHh1M/4YJKxcU/d3BkPfsYSSfqGRt/wPldUFtNUV8aaiVttaSoslpXRUFVCMMGfSF5o6UnwrV0n+bdXN3Ge9gamlTdBcNPFKzn05zNdQnlq5cWFlBcXznuXcnRsnIHhMSwCVSHNR+LuxAeTHO9KcLw7wYmuAY53JTjZk+D5Y938097X8Uk7NJXFhVzUUMmGhko2NlRy0cpKNjVUUV22NOZLETnXXz9xFMP4gxvXZ7qUrJY3QZDub8KFBRGqy8LtizczasqKqCkrYsuamjctHxkd51RskBPdCY53DXDkdD8vtfXx8L5T/N1zZ/ckmurK2NxYzZbGGjY3VnPJ6moqivPmT0OWqJPdCb61q4XfvLaJVdXaG5iJ/rcvYUWFEdYtL2fd8nLg7ERZ7s7p3mEOt/VyqLWXAy1xXjgR4+H9rQCYwbpl5axdVsa6ZeWsW1bG2uXlrFtWzuqaUooKNdhM0mtgeJQfHjrNjw+3Y5Y63FpTmhrAUV0apagwwuneIU7FhjgVG6Q1PsTxrgEiEeMPtqlvYDYKgjxkZjRUl9BQXcK2DSvOPN7ZP8yBljj7WmK8fLqf17oG+Ndj3QyMjJ1ZJ2JwXk3pmU7qpmVlrK0r57LV1aypK1UfhEzJ3ekeGGE06Osad2d8HMbdiRZGqCsretNAh5HRcX5ypIN/3vs6jzefZig5zorKYkqiBcQHk/QOJd9w6BNSAzjOqylhVXUpl66u4qaLG2ioLknjO81NCgI5Y3lFMTduXMGNG8+Gg7vTNTDCa50DvNaV6oc40Z3gRHeCx5vb6ewfnvT8Ii5fU8sVTTVc0VTD21ZU0D80Sk8iSXxwhJ6BJD2JEUqLCs5c6akkqlFOS9mR0308vL+VRw60crS9f8Z1S6MFZ4Zm15QWceD1OPHBJLVlUd5/VSO3Xr6aq5pqzwyzHh93+oZH6R1MMjw6xoqqktD645Y683MjNctt3brVd+3alekyJDAwPMqxzgH2tcR44USMPSd6eLVjYE7PjRisr6/gkvOquOS8atavKGdNbRmNtRoGm01OdCU41BpnRVUJ51WXUl9ZTMEU57yMjTuxxAit8SEebz7N9/e38nJ7P2Zw7fl1/PzGFVQUR4kYRCJGxIyCCAwnx+lJJOkeGKZ7IPiZSHLB8nLeu+U8brhwOdECHY5cKDPbPd3lgBUEsuhiiRFeOBnjRFeCqtJCasqKqC0roqY0Sm1ZEb1DSQ6einPwVG9wi3O6d/gNr7G8opg1daWsqS1j3fJy1teXc37Q36FvfenxWucAdz9xlH984XXGJp1PUxgxVlalDi0a0J1IDaGOD549VGMG16yr45c3r+LmSxtYUanDM5mmIJCs1xWch9ESnHh3sjt1Et6J7gSvxwbfcCx4eUUxGxsquXHjCm66eKVmkpynrv5hvvyjlyktKuTyNdVsbqxhVfXZc0wmB0BhxLj9urX8ypbz6B4Y5lRsiNZ4qjO2NTaEGdSWF1FXVhT8jFJXUcx159exokof/tlEQSA5bSg5xsnuBK92DnCsc4BjHQO8cLKHI6dTx5w3NlTyixev5BcvXsmFKyrpG0p1JPYOjdI3lDqG3N43zOneIVrjQ7TFB2nrHaKzb4TiaCR1FnhxIWXFBVQUF7KisoR3bajnXRfVU126CHsfr7wCn/88/O3fQn8/VFTA7bfDnXfC+vSObz94Ks6Or++mvW8IgORY6v9/fWUxWxqrKYkW8OiLbWcC4PfedYG+zS8RGQkCM7sPeA/Q7u6XTrHcgC8BtwAJ4Hfcfc9sr6sgkAnHuwb44aHTPHboNLte62aG2UAAKIlGaAgOaTRUlbCsopjk2Dj9w6MMDI+SGBmjf3iU410JugdGKIwYV6+r492bVvDuTStprC2lq3+Ejr5hOvtT81J1D4zQWFvKVWtrpx6r/uij8P73QzKZuk2IRlO3b38btm9f3IaZxvf2neKT395HbVkRX73jKjY0VNLc2sf+lhh7T8bY3xKnLT7Eb1y9RgGwBGUqCN4J9ANfnyYIbgE+RioIrgW+5O7Xzva6CgKZSvfACE8cbqe9b5jKkkKqSqOpnyVRqkoKqa8spro0OqfhrWPjzt6TPTze3M6Pm9t56XTfnGpYVV3ClU2pUVNb19Vx2VAnBZdvgURi+ieVlcH+/aHuGYyNO59/7CXuefIVtq6t5Z7br9SHfB7K2KEhM1sHPDxNEHwVeNLdHwjuvwRsc/fWmV5TQSDpdrI7wY8Pt9OTGKG+spjlFanbisrUnFSvdgyw50QPu4/38MKJGK/HBgH4yx9/lV/d8yiFY6PTv3g0Cjt2wN13v6XauvqH2XMixu7jPbT3DdFQVcKq6hIaqktZVV1CdWmUT//zizzxUge3XdPEn733Ep0QmKeyNQgeBv7C3Z8O7v8I+JS7v+lT3sx2ADsAmpqarjp+/HhoNYssVFt8iOeOdXHT9RsoHZzDUNqqKojHp12cHBunq38kdTiqb5iWngQvnIyx53gPr3Wl9jYKI0Z9ZTEdfcNvmjG3MGJ85r2XcPt1axf0viS3zRQEOXFCmbvvBHZCao8gw+WIzKihuoRbL18NQzMcEprE+/vBnVPxIV5q6+VwWx8vtfVx5HQ/p3uH6B4YedNzllcUc2VTDbdd08SVa2u5bHWqo3d83OkcGKYtnuoYP907xBVrarmssXqx36YsIZkMgteBNZPuNwaPiSwNFRXQN3v/Qn+0hOs/8xh9w2cPIZ1XXcJFDZVc2VRzZnr05RWpnxOHf6bq74hEjBWVJayoLGFz46K+G1nCMhkE3wX+0MweJNVZHJ+tf0Akp9x+O3zta28cLXSO8cJCmt/9Xm694jw2NFSdmR58UYatisxRmKOGHgC2AcuB08BdQBTA3e8Nho/eDdxMavjoh6bqHziXOoslZ7zyCmzenPFRQyKQoT4Cd79tluUOfDSs7Ytk3Pr1qfMEZjuPQCEgGaZxZCJh2r499Y1/x47U6KBIJPVzx47U42k6mUxkJppiQkQkD8x0aEh7BCIieU5BICKS5xQEIiJ5TkEgIpLnFAQiInku50YNmVkHEAOmm6WrepplUz1+7mOT75+7bDnQOd96ZzFdrQt9zkzrzLV9ZmuvfGqf+f7tnHs/jLaZrq7FeE6+tk+6/m9N9Vg62metu9dPucTdc+4G7JzvsqkeP/exyfenWLYrne9jIc9ZjPaZrb3yqX3m+7czRVstetuofRa/fdL1fytb2mfyLVcPDX3vLSyb6vFzH/veDMvC8Fa2MZfnLEb7zNZe+dQ+8/3bmWsdC6X2mdl8t5Gu/1tTPZaJ9jkj5w4NZYqZ7fJpTsYQtc9M1DYzU/vMLB3tk6t7BJmwM9MFZDm1z/TUNjNT+8ws9PbRHoGISJ7THoGISJ5TEIiI5Lm8DAIzu8/M2s3sxbfw3KvM7ICZHTWzL9uk6wWa2cfM7LCZHTSzv1zcqtMjjLYxs8+Y2etmtje43bL4ladHWH87wfI7zczNbPniVZxeIf39fNbM9gd/O4+Z2XmLX3l6hNQ+nws+d/ab2T+aWc18XzsvgwC4n9SV0d6KrwC/C1wY3G4GMLMbgVuBLe5+CfA/F15mRtzPIrdN4Ivufnlwe2RhJWbU/YTQPma2BrgJOLHA+jLtfha/fT7n7pvd/XLgYeDTCy0yg+5n8dvnh8Cl7r4ZOAL88XxfOC+DwN2fAronP2Zm683sB2a228x+amYbz32ema0Cqtz9WU/1sn8deF+w+N8Df+Huw8E22sN9F+EIqW2WjBDb54vAfwZyevRGGO3j7r2TVi0nh9sopPZ5zN1Hg1WfBRrnW1deBsE0dgIfc/ergP8E3DPFOquBlkn3W4LHAC4C3mFmz5nZT8zs6lCrTa+Ftg3AHwa7rveZWW14pWbEgtrHzG4FXnf3fWEXmiEL/vsxs/9mZieBD5LbewRTWYz/XxP+HfDofAsI7ZrFucTMKoDrgW9NOmxbPM+XKQTqgOuAq4F/MLMLPMfH5y5S23wF+Cypb3KfBT5P6g825y20fcysDPgTUoeFlpxF+vvB3f8U+FMz+2PgD4G7Fq3IDFqs9gle60+BUeCb832ugiAlAsSCY5BnmFkBsDu4+11SH2iTd7sagdeD31uAh4IP/ufNbJzUZFEdYRaeBgtuG3c/Pel5/5vUcd6lYqHtsx44H9gXfBA0AnvM7Bp3bwu59nRYjP9bk30TeIQlEgQsUvuY2e8A7wHe/Za+fIY9mVG23oB1wIuT7j8DfCD43Uh1+k71vOdJfes3UrtgtwSP/z7w58HvFwEnCU7Yy7VbCG2zatI6/wF4MNPvMZva55x1XgOWZ/o9ZtOxraYAAAM/SURBVFP7ABdOWudjwLcz/R6zrH1uBg4B9W+5pkw3Sob+IR4AWoEkqW/yHyb1rewHwL6gUT89zXO3Ai8CrwB3T3zYA0XA3wbL9gA/n+n3mUVt8w3gALCf1LebVel6P7nQPuesk9NBENLfz3eCx/eTmoxtdabfZ5a1z1FSXzz3Brd751uXppgQEclzGjUkIpLnFAQiInlOQSAikucUBCIieU5BICKS5xQEsiSYWX+at/fMIr3ONjOLBzNrHjazWScrNLP3mdnFi7F9EVAQiEzJzGY8697dr1/Ezf3UU2eWXgG8x8zePsv67wMUBLJoFASyZE03q6OZ/UowOeALZva4ma0MHv+MmX3DzH4GfCO4f5+ZPWlmr5rZH0167f7g57Zg+beDb/TfnDRP/C3BY7uD+eNnnFrD3QdJnRA0MRnd75rZv5rZPjP7jpmVmdn1wHuBzwV7EevnMnulyEwUBLKUTTer49PAde5+BfAgqemfJ1wM/IK73xbc3wj8EnANcJeZRafYzhXAJ4LnXgC83cxKgK8C24Pt189WbDAr64XAU8FDD7n71e6+BWgGPuzuz5A6O/uTnrq2wyszvE+ROdGkc7IkzTKrYyPw98Ec70XAsUlP/W7wzXzC9z11jYlhM2sHVvLG6YABnnf3lmC7e0nNJdMPvOruE6/9ALBjmnLfYWb7SIXAX/nZyeYuNbP/CtQAFcC/zPN9isyJgkCWqilndQz8L+AL7v5dM9sGfGbSsoFz1h2e9PsYU/+fmcs6M/mpu7/HzM4HnjWzf3D3vaSuZvU+d98XzC65bYrnzvQ+ReZEh4ZkSfLUVa2OmdkHACxlS7C4mrNT+P52SCW8BFxgZuuC+78x2xOCvYe/AD4VPFQJtAaHoz44adW+YNls71NkThQEslSUmVnLpNt/JPXh+eHgsMtBUteUhtQewLfMbDfQGUYxweGlPwB+EGynD4jP4an3Au8MAuS/AM8BPwMOT1rnQeCTQWf3eqZ/nyJzotlHRUJiZhXu3h+MIvpr4GV3/2Km6xI5l/YIRMLzu0Hn8UFSh6O+muF6RKakPQIRkTynPQIRkTynIBARyXMKAhGRPKcgEBHJcwoCEZE89/8B5ILyATniWkgAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light",
      "tags": []
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "learner.recorder.plot(skip_end=10,suggestion=True)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 325
    },
    "id": "Is4oFNhKo90-",
    "outputId": "c1bab844-bf58-4223-ee36-19d88ca8709b"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>error_rate</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>0.414978</td>\n",
       "      <td>0.406218</td>\n",
       "      <td>0.846622</td>\n",
       "      <td>0.153378</td>\n",
       "      <td>1:34:34</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD4CAYAAAD8Zh1EAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3dd3gU1foH8O+bTqghhBo0oQdpQoAoRQSUACpWBMUGXq5eu16vwXYFQUC9+FMvdrl2QbGA0gSk99ADoQSIEGpooaaR8/tjZzezu7Mt2WSS5ft5Hh92Z2ZnzuzGd86c854zopQCERFVfkFmF4CIiPyDAZ2IKEAwoBMRBQgGdCKiAMGATkQUIELMOnBY1ZqqXUJzsw5PRFQprV+//rhSKsZonWkBPaJ2faSmppp1eCKiSklE/nK1zrQmF6a/ExH5l3kB3awDExEFKBNr6AzpRET+ZFobOhGRrwoKCpCVlYXc3Fyzi1LmIiIiEBsbi9DQUK8/Y1pAZwWdiHyVlZWF6tWrIy4uDiJidnHKjFIKJ06cQFZWFuLj473+HNvQiajSyM3NRXR0dEAHcwAQEURHR/t8J8I2dCKqVAI9mFuV5Dw5UpSIKECY2uTCWjoRVSanT5/GBx984PPnBgwYgNOnT5dBieyZWkO/VMSATkSVh6uAXlhY6PZzs2fPRq1atcqqWDampi0WFimEBJtZAiIi76WkpGDPnj3o0KEDQkNDERERgaioKOzYsQO7du3CrbfeigMHDiA3NxdPPfUURo4cCQCIi4tDamoqzp07h/79+6N79+5YuXIlGjVqhBkzZqBKlSp+KZ/pAZ2IqCRG/7YN2w+d8es+WzesgX/ffJXL9RMmTEBaWho2bdqExYsXY+DAgUhLS7OlFk6ZMgW1a9fGxYsX0blzZ9xxxx2Ijo6228fu3bvx/fff49NPP8XgwYPx008/YdiwYX4pv6kB/dIlBnQiqry6dOlilyf+3nvv4ZdffgEAHDhwALt373YK6PHx8ejQoQMAoFOnTsjMzPRbeUwN6AVFRWYenogqMXc16fJStWpV2+vFixdjwYIFWLVqFSIjI9GrVy/DPPLw8HDb6+DgYFy8eNFv5TG1U7TgEgM6EVUe1atXx9mzZw3X5eTkICoqCpGRkdixYwdWr15dzqUzuYaeV8CATkSVR3R0NLp164Y2bdqgSpUqqFevnm1dcnIyPvroIyQkJKBly5ZISkoq9/KJp1xwEZkC4CYAx5RSbQzWC4B3AQwAcAHAg0qpDZ4OHN6gudq6aQNa1KteooIT0eUnPT0dCQkJZhej3Bidr4isV0olGm3vTZPLFwCS3azvD6C59t9IAB96VVIA+YWsoRMR+YvHgK6UWgrgpJtNBgH4SlmsBlBLRBp4c/A8BnQiIr/xR6doIwAHdO+ztGVORGSkiKSKSCrAGjoRkT+Va5aLUuoTpVSitf0nn1kuRER+44+AfhBAY937WG2ZR6yhExH5jz8C+kwA94tFEoAcpdRhbz7IgE5E5D8eA7qIfA9gFYCWIpIlIiNE5BEReUTbZDaAvQAyAHwK4B/eHvzkhfwSFJmIqHKoVq0aAODQoUO48847Dbfp1asXUlNT/XI8jwOLlFJDPaxXAB4rycFf+TUN9yVdWZKPEhFVGg0bNsT06dPL/Dh8YhERkZdSUlIwefJk2/vXXnsNY8eORZ8+fdCxY0e0bdsWM2bMcPpcZmYm2rSxjMu8ePEihgwZgoSEBNx2221+ncvF1KH/REQlNicFOLLVv/us3xboP8Hl6rvvvhtPP/00HnvM0ijxww8/YN68eXjyySdRo0YNHD9+HElJSbjllltcPhP0ww8/RGRkJNLT07FlyxZ07NjRb8VnQCci8tLVV1+NY8eO4dChQ8jOzkZUVBTq16+PZ555BkuXLkVQUBAOHjyIo0ePon79+ob7WLp0KZ588kkAQLt27dCuXTu/lY8BnYgqJzc16bJ01113Yfr06Thy5AjuvvtufPvtt8jOzsb69esRGhqKuLg4w2lzywPb0ImIfHD33Xdj6tSpmD59Ou666y7k5OSgbt26CA0NxaJFi/DXX3+5/XzPnj3x3XffAQDS0tKwZcsWv5WNNXQiIh9cddVVOHv2LBo1aoQGDRrg3nvvxc0334y2bdsiMTERrVq1cvv5Rx99FA899BASEhKQkJCATp06+a1sDOhERD7aurW4M7ZOnTpYtWqV4Xbnzp0DYHlIdFpaGgCgSpUqmDp1apmUy7Qml1qRoahTLdzzhkRE5BXTAnqwCB9BR0TkR6YF9CARXCy4ZNbhiaiS8vSUtUBRkvM0NaDnFxbhUtHl8eMQUelFRETgxIkTAR/UlVI4ceIEIiIifPqcaZ2ipy/mozaAN+ftwKj+l88zAomo5GJjY5GVlYXs7Gyzi1LmIiIiEBsb69NnTAvo1sfPfbxkLwM6EXklNDQU8fHxZhejwjKtyaV6ODMmiYj8ybSAXrcGUxaJiPzJtIAeGcYaOhGRP3EuFyKiAMGATkQUIBjQiYgCRIUI6IWcAoCIqNRMDegPdYsDAOQzoBMRlZqpAb1xVCQAIL+QAZ2IqLRMDegL0o8CALYfOmNmMYiIAoKpAX3HkbMAgIzsc2YWg4goIJga0N+4rQ0AIKFBDTOLQUQUEEwN6FGRYQDYhk5E5A+mBvTw0GAADOhERP5gakAPC7YcPq+QTy4iIiotk2vo1oDOGjoRUWmZG9BDGNCJiPzFq4AuIskislNEMkQkxWD9FSKySEQ2isgWERngzX7DGNCJiPzGY0AXkWAAkwH0B9AawFARae2w2csAflBKXQ1gCIAPvDl4eIilUzSvgG3oRESl5U0NvQuADKXUXqVUPoCpAAY5bKMAWJPJawI45M3BrU0unMuFiKj0vAnojQAc0L3P0pbpvQZgmIhkAZgN4AmjHYnISBFJFZHU7OxsW5bL7K2HfS03ERE58Fen6FAAXyilYgEMAPC1iDjtWyn1iVIqUSmVGBMTg6AgAQCkHeRcLkREpeVNQD8IoLHufay2TG8EgB8AQCm1CkAEgDr+KCAREXnHm4C+DkBzEYkXkTBYOj1nOmyzH0AfABCRBFgCerY/C0pERO55DOhKqUIAjwOYByAdlmyWbSIyRkRu0TZ7DsDfRGQzgO8BPKiUUmVVaCIichbizUZKqdmwdHbql72qe70dQLeSFOC6FjE4fSG/JB8lIiId058pGhYSxIFFRER+YH5ADw7CKdbQiYhKzasml7I0S8tBP59XiKrhpheHiKjSMr2GbsVmFyKi0jE9oEdoU+jyIRdERKVjekB/47a2AIBcTtBFRFQqpgd064yLuXxqERFRqZge0AuLLE0tJ84x04WIqDRMD+jWtvPME+dNLgkRUeVmekBPaGCZRj2mWrjJJSEiqtxMD+jWLJdcZrkQEZWK6QHd1inKLBciolIxP6BrNfQdh8+aXBIiosrN9IAeEWqpoU9ZsQ9Ld3EKdSKikjI/oGtNLgCwaOcxE0tCRFS5mR7QQ4PF9rrgEjtGiYhKyvSALlIc0Hs0jzGxJERElZvpAV2PNXQiopKrUAE9t4ABnYiopCpUQL/IXHQiohKrUAE9N58BnYiopCpUQB83O93sIhARVVoVKqATEVHJMaATEQWIChHQvxreBQDQo3kdk0tCRFR5VYiA3rNFDNrF1kRwkHjemIiIDFWIgA4AIUGCwkvK7GIQEVVaIWYXwOromTwcPH3a7GIQEVVaFaaGfvD0RbOLQERUqVWYgG61JYu1dCKikvAqoItIsojsFJEMEUlxsc1gEdkuIttE5LuSFmjwx6sQlzKrpB8nIrpseWxDF5FgAJMB3AAgC8A6EZmplNqu26Y5gFEAuimlTolI3ZIWiBN0ERGVjDc19C4AMpRSe5VS+QCmAhjksM3fAExWSp0CAKVUqR89dPxcXml3QUR0WfEmoDcCcED3PktbptcCQAsRWSEiq0Uk2deCDE6MtXt/kRN1ERH5xF+doiEAmgPoBWAogE9FpJbjRiIyUkRSRSQ1O9v+gdD92zbwU1GIiC5P3gT0gwAa697Hasv0sgDMVEoVKKX2AdgFS4C3o5T6RCmVqJRKjImxf9xc20Y17d7nFbItnYjIF94E9HUAmotIvIiEARgCYKbDNr/CUjuHiNSBpQlmb2kKNnOT4zWDiIjc8RjQlVKFAB4HMA9AOoAflFLbRGSMiNyibTYPwAkR2Q5gEYDnlVInfCmI47D/9/7MwK6jZ33ZBRHRZU2UMmf+lMTERJWammp7r5RC/KjZTttNGtweN7dviNDgCjcGioio3InIeqVUotG6ChMlRYxnWnz2h80Y/sW6ci4NEVHlU2ECOgBsH9MPE+9o67R82e7jJpSGiKhyqVABPTIsBLd3jPW8IREROalQAR2Ay7byd+bvwoX8wnIuDRFR5VHhAror7y7cjYHvLTe7GEREFValCegAsO/4ebOLQERUYVWqgG7k1Pl8vDZzG8xKvyQiqigqZEDfOTYZTWKqGq57aupGu/dXvz4fX6zMxLsLd2P+9qPYfMDcB2Scyys05SEdR8/kIudiAQBgT/a5cj8+EZmvQgb08JBg/PlcL8N1MzYdMlz+fwt2429fpWLQ5BUAgJwLBfh6VWa519yf/H4jbvnvCpzPc+7AXf/XKVvQLY2MY/YB+825O9D1jYXoPvFPvPJrGvr8ZwniUmZh2e5sF3uoGC4VKWSf5TTJRP5SIQO6N9buO2n3ZKOQIPuBSU9N24hXZmzDpnKusf+5wzIV/HmHjJzCS0W448OVaD/6Dzz85boSX2jemb8LfSctwTvzd9mWfbZsHwDgbG4hvl79l235fZ+vtR27vDOEMo6dxaT5u9ye55vzdqDzuAU4wbnvifyiQgf0KQ8m4um+TpM2ArA8qk7v+X4t7d4v3mmpnd72wUpMmLOjbAroRp7Dk5cKi4oD24L0Y8i/VLLZJN9duNvuXwBQcB00Cy4V4elpm9D61XklOl5J3ff5Wry3cDdOXXB9RzI37QgA4Ewu01GJ/KFCB/Tereqh4xVRTsuNan3r/zrlcj8fLdnjtCz98BlbQAGAIzm5WO7DiNQlu7KdmjT26tqu8wotD+g4cPIC1u47iSKHMp/1YxAruOQ6oP914gJ+33K4RPvNPH7eqXnH+zJZLlgXC1w/qOSvExcAON9dEVHJVOiADgBX1I50WvbqjG1Oy6xNHQA8PmT65Pl89H93GR75Zj1+22xpk7/9gxUY9vkar8v1wJS1tiaNBduPYueRs+j9nyW29V+szAQA9HhzEQZ/vAqXiuyDbnk9kUn/KD9XzR/7jp/HLxuznJb3ensx+k5aYvAJb46bDwB4e95Op3UFDncnB05eKNExiMhehQ/ocXWqYtm/rrdbpm8ntiosct8mffD0RRw/l4eciwXo+Pp82/K5244gLmUWDuXkAoBXtfQXf9lq9/7hr1LR7/+W2i37ZvV+7D9RHKhW7z1pt77gUhGOnc11e5wN+0/h5veX4/SFfI9lcuXomeJjOF5UrMuuf3sxnpm22W75ygz/zJ/jOAXy3LQjaP7SHCzSXYDv+cz7CykRuVbhAzoANK4did+f6F6qfUycswOJYxeg87gFdstnOTRHWGvpSikcyXEOuH//OhXfrdnv1TF7vrXI9vpvX6Xarbv3szXoMm4hjuTkYsamg+jzn8VONejbP1iJrQdzMOzzNXh22iakHcxxOoanzlV9QDdq/li7r/hCY93XsbO5dkG2yM3FcsC7y/C0QyqpnuOTp/7YZmnmmrnZPlspLmUWZvjpoSZxKbOc+k027j+FwzkXvd6HmXcN2w+dweKdpX7OeoUwaf4ujJ+TbnYxLhuVIqADQBuHR9T5yhpA8r14tF1RkcLMzYeQNH4hNu63b5uft+2o3Xt9wPTFYe1ikTR+IZ6augl7ss/b7jJ+33LIru067eAZ/LzxIG5633nqgw37XfcdWMpX3OSyJcv9BaHly3MRlzILGUft282HfroaGcfO4c8d9ud+4lweth8+g18NUkk7x1n6Pm7vaP888SuiLU1oh047B9enpm5ySvcsaRu+Y7/JbR+sxDXj//Tqs79uPIgeby7y212Krwa8twwP/i8wpox+b+FufLykVA8vIx9UmoAOAH0T6pbLcV6ZkYZJWlrgfq2mVnipCJ8tc/7D7PrGQr8dt+XLcwAAj3+30a7t2lWnYZdxC7Ag3X1N7ojughMeUvxz5xZcwrEz9jVxa+ZNnkMb95p9J9F30hIM/6L4LiOv8BL+NX2Ly+Ouy7RcaIIc5rkXiG2fRlJ+tjRnKaXw8Jfr0HfSEszbdsRw25I4aHAhcbRZGxi2OSsHz/+4GfdPWeu34/vCU5Nceflw8R6M4DMJKoVKFdB7NI/xvJEffLtmvy0DI/P4BczbdgSvzEjD2Fmlv3X8+R/XulxXpIAWWlDXc9U/cOxsHj5c7JzBY7eNLqDf+dEqZByztGkPeG8Zuri4GNWtHu5yf9ZmkZYvz8XCHZ6bBXYdsW9DP5PrfmDVb5sP4c8dR/Hx0r22i9W2Q2dcbr9o5zFbRhFg3zyUc6HAqUlt0h/F+fs3TFqCh7+0bwoDgDBtxs+Jc3fgx/VZWLrLPpsp49hZw/4IwHLH1vvtxX5psukyzn+VhdKYOHeHV7+12eamHbnsR0mHmF0AX7z9h3PGRFl7Z4ElAFQP989X1axuNbfrvWkS8uT+a67EV6ssHcfWmrJV30lLkTlhIPZmu57oLLfAdRmemrrJtm+9/63YhyNncrHn2Hm7wVw/bzyI37Ycwu5xAwBY7nQ80d8JALAL2FZpB3Nw8nw+HvrfOtSpFo7Ul/sCgF1+f/sxfwAArowu7n/RN+nsPnYOuw2adDYbTN1QVKQQFCT468R59J20FH+/rglG9U9w2m7augPYe/w8pqzYh3/ffJWnU/Wb9MNnEF+nKiJCg23LvluzH1uyTmPCHe3KrRxmeuSb9QCAzAkDTS6JeSpVDf2R65raXq97qa/H7d39sI6ZM56cNRjKXxL6Zo+yEhYchB7N67hc7zgfjqM8N7njgHHO/+jftuPjJXuxIP2oXaokYMmTj0uZhfTDZ9Akxv0FzcjHS/Zi6tr9iEuZZUtJven95bamEP3xRv+23enzP20oTsm8rqXnuzzHjCQA2Kp1SM/aethWprs/XoVch+/K2lTny4U552IBWr0yx22n7XM/bMY9n652+fn+7y7Dcz/aZyq9+MtWTF13wGn7oiLltqPblUCepiHr1AWn37IyqlQBPblNfdvrGDfNAnrbRvczXB4ZFozJ93T0S7l8EVYOD7uuEhaMrvG1Xa53NR+OVa5Bjdgfbnp/uWFe+ryne3r8rLVtHbDPzLH6MfUA4lJm4ffNzudWIyLU9nrUz1udOlrP5RXaApyrQHxJ6zx+c25x+dfsO+kURK1cxcuVe45j0OQVdrn47Uf/gdyCIlwz/k/DtNnss3n4aUMWVu45AcByl7Ng+1Fbh/bXqzIBAKu09QDsBr3FpcyyuzNq8uJsNHlxNtqP/sPlRcJI+mHXTV/+ll9YhPFz0j020fmDUgrdJy7Co1oNvyLILbhUoulBKlVAr+ah2eN/D3V2WlY1PMSwuUQBGNiugb+KZvN8v5b4ZkRXv+/XG9c2jQZgqRH/o1ezEu/nYn7pm32MXCpSaNfYPltp4h1t0bJ+dZ/24zjtAwA8r3XQGt1J6adJAIC+k5bY1VDb/Hsemrw4GwDw2m/Og9YA1x3T1jb6uz9ehRvfKe7IvlRk/B3+a/oWbD5wGodPG3d4Gg1u06f9zU07gmYvzcHDX6UifpSlzG9r/QInzxePV7AOerP6eKlzh37OxQKs3HMCj3ztXSBzHFPgC18H0v175jZ8vGQvxs8u/bQdy3ZnIy5lFuZsNR4xbe2jWrSzdJPZvTB9i+0u0nHwnC/SD59Bq1fmImn8Qp/viipVQK9XIwLTH7kGW1+70XB9w5pVDJdvNail144M82vZrGpFhqK7i+aOIAFEymaYe8t61W0pjB8t2YOgUgyn/++iDNvr+jUiSl02vRUZJ+zeJ7exXFRDg8t3+L+ruXSMcv0B4Jb/rsBj320w3ldhEdbsO4ldunTPH1KdR94CxYO7er61CHEps1weDwBqRFgqIvppIh7xUIucsnyf4XJ3U2NYB9fd/sEKt7XCsbPS8dGSPViw/SiavzQbs7YctmsGc6S/2xjuY5bM92stYz32Hbe/m3rs2w126bPn8wpxqUg5dVLnFV7C92v3Qyllu7g9+u0Gw0y1QjdTZ/hiWmpx89aZUsyq+vrvlmbDo2fy0HncAp9q6pUqoANAYlxtVNduobeN7ofpj1yD6KqW4FwrsvjWeu8bA9zuxxrw/tYjvkTlSB+TbLg8NMj1V7prbP8SHcsbt3Ro6NSZ6W2zlCP9rfXoQWXbsRcZZunEc0xvLGuu0iaNcvWtHDNmrFxlGhkF68MOg9WMxhZYWefomb/9qMttHI35fbstk0nPOtum0WA5qw37T9v6CKwcg/WEOTvw8FepKLik7C5wD/1vrVOtVH+3sWqv/YXcW/r+DKUUZm09jOFfpCL7bB5+2ZiFq/49D8O/WIc5acXlzrlYgPGzd2DUz1ttk/RZGWWq6S/uu0txF6LnWHHzZbzKiXP2I8O3urnoO6p0AV2vangIEuNq29VGq4YF48nezbyuob40sDWa1DF+mIY7VcKCcVenWKflwdpxn+jdDJ8/kIi1L/WxrQtx035uvSh5o1Et5zuRGhEheOtOSzbD1JFJAIAezVx3jOpNeTDRcPnInk1wY+t6bj/72PVN3a53p99V9WwPBS/vCbr2+3EkqDUTytHEuSVvLhjUoSEuFlwq0cNS+k5a6rRs9d6T6DJuAd6Y7T719vHvNiLr1AV8atBE486indlefacfLM5AXMosZJ3y/fvXD5LrPG6BbbqKJbuy8fh3xR39mcfP2+ZScncHBFgucJm6R1ve8I7zd+fKtkM5LmvP+uWbDpxG1zcWYvp657u2/EJLf4jV6Qv52OlwUVm2+ziW7z6ORV6MHq7UAd3KGrjCQ4KwbUwynr2xpdM2Y29t4/LzX43ogmrhIXjdx9rogLbObfAhWtPBcze2RJ+Eeqhb3dJk8cA1V9q2cQzeqS/3xfIXejvta+rIJKx9qQ/mPNUD/3vQ0j/wxUOdMf9Z507EDo2jcFdiY2ROGIikJpa29Ete3qr1bmUctK9pGu2xiejapt5dNIzo75Kf7tuixPtxx1VOvfW2Vu+vE/59Zu0yH2bvdGStST/6jXEzj5V+viBPjp3Nc5pywUj3iYswbna6T1MlAJZO2ZUZx902w1g7lbtPXGQLtpsPnMaF/ELsO34evd9ejOPn8gwHEXr7wBbrQ24AoJaHilLS+IV223tjwfajiEuZhYHvLcevmw7iXF4hVjvcgbz/Z4atvPO3WwbG/fPHzU5TOrz9x048/FWqbW6jHhMXwdFb83Zi2Odr8JAXo4cDIqCPv6MtFjzbE7XctIvf4KamGRsVibTR/RCrm9nxs/uNa60AkNCgBgCgl0EKnFFaYuaEgRg9qPiCcuK8/S1VdNUwVAkLxo7X7ZtxYqOqoG71CCQ0qIHrW9VF5oSB6NWyLiLDnDt528Y6T43gKZvFE8ca/oC29Z22KU1Lib5j8m89m+Aa7UIEAJ8/4Pr798UxF51KRtks17212C/H1Nt99Cy2ZuXg9y2+/RbWJiFPI1unrDBuM/eH027msjfy8q9pPk20dtP7y3HqfD4GTV6BzmMX4Pq3F2Pv8fOYk3YEV9T2/a7ZyMQyeBbCqzPSbK9/3nAQbf49D0M+sc8W+mJlJu77fC0+X74PkxcVN8k5TulgnVrkIa2PobTp0QER0MNDgtGsrvtMiXo1ItDvKktQH9rlCsNt9INt+hpcAKzZMtW1zioRQdtGNW37BWBrQnCnu0OgtNaCHS8G7mrH20b3s11YnujtfUZL6st9neZXccXaRNRcGwz1wb2d7Nan9G/lVEPseEUtr8tSLcL+wvTtw12RPiYZa17sgz4J7pt6AGBYkvHvaDXQ4A7KW8FBgq9HdMHqUX08b+zGDe8sxc3/XY7Hv9uItq/5/yEj1qaFstD/3WV+3Z9RrX2Mdqd0XpcFEyTAkl2WGmtEaJCtT6BuCTrozxkEyFPn8932JVidzS3Ap0v3OuXsH9J91tNdmNGdoF6+rkN2/V/G/Tq+CIiA7i3rxFo7jxjn0+priHrWuDrt79cAAG7SpTv+9kR3fHxfcW3SaNIpR64GFzkGcHfNylXDQ/BQtzgAQP82ngOXNeUzLCQIkwZ3wOwne2B4t3i8cVtbw+1HdC/uLJ7/7HWGg7Ruv7oRBic2RtWwYPz+RHdkThiIqroU0WFJV7jMXnkhuRXG3GLfDBYUJKgSFox62v+4/7mrPd4d0gHrXuqLL4d3wR/P2Dc1jb21rcvBY78/0R3vD73acJ03Ol0ZhR7NY1C/pv+yfHx5qMm/b27t075vu9r4Il2ai1p5cGyqAICXfknDHq1ylVtQhL6TlmLj/lN4ZtomvxxzxJfrkDR+od002kZG/7Yd42anY9HOY4hLmYUbJi3xS158amZx4A7XVQCN2tgdeXrWw2UV0K2s7dqOWjesgW2j+9kyZDLG9ceO15NhDUlN61ZF+phk3Jd0peHnAWCpF+2m1g7bF5Jb4aNhnZzW168RgYe7x3tMGbyrUyzWv9wXrRvWMFxfQ1cDvrerpTZrvZi0blgDr97cGvdoyx07Whu4CGRfDe9iex0eGoygIMG2Mcm22TDfvqu9bf3YW9ti97gBhkH30V5NUVOXlWTkjk6xGNShEWKqh+O6FjFoUc/4Lsw6gjg2ynIOr97UGm0a1XTqGH/ShzuZ8hjRq+c4VuKhbr5lX/2y8SD6GtzVjLvNdd8RUDx2wSyOWT+u3PXRKrs8+9LYsN/S0expf9YAO0Kb72f3sXNo99ofpT7+nR+tsk2Hve1Qcaft92uL0x5LmiNwWQX0tNH90L1ZHbw7tIPLbaqGh9gCQUhwECJCg9HvKkvbcWhQEKqEBbttCvGm+cP6Y8XXibQb/QpY2ttXv9gHL9/U2mOHpIgguprr1MTbOxZn4aT0b4VdY/sjPCTYcNvGtRnufDEAABEmSURBVO0DuquJyHq2KO43iAh1/vOp5+e8dW+k9G+FzAkDsfyF3sicMBDDdXcXC5+7zvb62Rtb4v/udv3b6/1Hd2HyxjO6Tl3rvDKe6EfIbh3dD1dGOz+dy1u9WsagSpj9b7vxlRvc9isFBwnitAyvj4aV/6hpX3h6gI0/5VwowAtuZhL1B+t02OddDLiqWsK5o7wK6CKSLCI7RSRDRFLcbHeHiCgR8U+Plp9VCw/BNw93dRnUXPm/IR2walRvr1Ih2/owb3sJRvb65J+6B2eLCMLc1Do/vLcT3h3iXbCzcjWNQeaEgU618gevjQMA7BybjIxxJc/Ht6Y3/vNG77JimsZUwxu3tcWS53sBAHp7OQWzvr32vaFXG45C1nvs+qbo1TIGU0cmoU61cCRe6fwsXEct61dHm0Y1ME1LM7XWsG8yGMEc7yG1ttMVUba0VasoLcPj1g4NDT9TpBReHJCA8be3tVVa/K00aa3+Nmmwdxfp/u8utRskZAZXzb+eeAzoIhIMYDKA/gBaAxgqIk4NfCJSHcBTAALueWLhIcFo4GIUqpW1o9WbkaAp/RPQrVm0XW23LHiaKkEvqmoYBnUobod11Sar58uo11dvam27Q3CXj+/JnVruv6tmJiP3dL0CV0ZbAmJJ5tK5pX1DXN+yLvaNtzTFVQsPsauRT3kwESHBQfjioS62lNEcDyMFrYPgfn+iB7pqn7GmvF7V0LlSEOPmTgwARvSIt5tpUe+duztgaJfGACxNSdYLe0RIMKqFh2Bolys8/pYl/Vsd0b1Jqfoy/OWbEV1dNtk5OuRlM1BZslaAfOXNX3cXABlKqb1KqXwAUwEMMtjudQATAZj/bZjgjdvaeBydahVfpyq+fTipxLdVvjAahOSNVj7Or+JJUJD7OwRvvXxTa7w8MAG9WpTsYScRocFuZ6J0R0Sw8ZUbsPrFPniqb3M8368lkq+qb5jH7y7bactrN2LDyze4XK9guXXTXzRe6N/K9vofvZpiZM8mdp8xSmXVl9vaPls9IgS7xvbHv5JbOs3N/+Mj1+COjrF49gbnu5+lu7Kx8ZUbDO9A9c1ajqqFh+Dm9g3Rop7vs2y6k6L7PjwJCwlCt2bRtqdlGZk2MskuEaC0vn3Yfj6n+c/0xKpRzmNNXLm2WR2sHtXH48A+R95ElEYA9PcfWQDsSisiHQE0VkrNEpHnXe1IREYCGAkAV1zhPuWsshGRUuVkl5W5T/fABR8nRgJg1w7tKPmq+obpYOWhWngIHu7RxPOGbnw1vIttYqvMCQMx6Y+deO/P4vlr3F3MonQDVR673nV/SYiL7J4u8bXtZn/UE9h/5qm+zfFU3+a299tG90MVrSPa03Nta1YxPsZxbVi50eRtneNqo3NcbWSfzbNNA2zVqn51RFUNw7S/J2HT/tPYd+I8XvrFko/dNKYa1r/cF53GWp7X+8pNrZHcpj7+OnHedhGPrhoOwDIvy/UtY6AAp2H5b93ZzjbJmjuzn+yBhAbVnZ4b68rW126EiLj83utWD0fXJtFY4adHDm557Ua7Y00bmYTm9ar7PHti/ZoReLx3M/zhw9QPpa4yiUgQgEkAnvO0rVLqE6VUolIqMSamfJ4+dLmrHhHqU0flon/2wq+PdXNbw/zovk745mFzZpT0B8fmhX9c38w2pwwAWzpoabgKHj9oqa9GorRmGFef1XfYt9MNJLNm9+hteMX+DiDYh7QJ/Xfx2f2J+HJ4F/z06LXauhBc26yOrUnOOhZC3znf8YpaaFSrit0o4om6h2y8f09Hw+wuxyeSGd1JrX2pD1o3rOFTc5+7+ZUAy988ANQp4dxHjqy/3997NkH18BBbk5qIuJ3W2oj+jiipiefPelNDPwigse59rLbMqjqANgAWa19yfQAzReQWpZTz872oQvPU+RYoPhrWCeFalk5EaDC2j0m25fgOTmzs7qNesQbFD+7tiHaxNbH76DnDJyHpDe8ej0itTdsTfXPd939Lsr2+tUND/LrpkFMAf+vOdnj2B+O5212VHTAeYGfZJgSbX70RkeHO7fZG9VBrc0e18BBUCw+xmx3x0V5N8UKycxOKY59A05iqdinHO15PxsL0Y3aThD1yXVPbA8J3jk3GqfMFLpMZ1r7YB9HVwm3fVRcfg22LetXsZthc9q/r7e7MRg1IwKgB9k+1mvb3azDsszVY7uZuoI7u4qi/cH08LBHP/bgJn7spkzc19HUAmotIvIiEARgCYKZ1pVIqRylVRykVp5SKA7AaAIM5VWjJberj+pbG7fD+mOLY+j/s0TO5iI2KxPWt6nqcryY0OAj3JV3pVW1aXytvrJuy4u272hs+1KWXdq5DOnu+WFnP/yoPHc81I0Pt7uRaap2Orp6G9P7Qq/H7E5bHAerP0SiYA3Dqc/mnwxxNEaHBGNiugS1LCLDUiq3HCg8JdjswrG6NCLtytKpfA50MspMyJwxEQ91+nunbAt+M6IrvdBfSyfd0ROPakR6TJwB4TE91fOIXADSpUxU1I0Px2QPus608BnSlVCGAxwHMA5AO4Ael1DYRGSMit3j6PNHlyDoAqyweogK47nQNCQ4y7GyvXTUMK1J6u52kTm/u0z3sApY3rtEGKbnKfb+5fUNb3rs3nruhBTo0Lp5KwlUqetcm0Ugb3Q+rR/VBVNUwZE4YiJvbG6dqPtmnueFyq2kjk+yas6z+1JplACDtUA66N69jV5OOrub9bKm1XUwY5qqpb/kL1+PXx7t5tW+v0iyUUrMBzHZY9qqLbXt5dWSiCmbS4PY+pXq6kxhXu8wfVrzon718eqKNLxlPrep7nxZq9eKABNzcvqHHB6Fbjb+9rW1eJKs372iHf/1k6RhtElMNvz7WzdYUVreG6zZua1OOJ8/e0ALDu8W5TPEMCQ7CzMe7I37ULLtxIvrt9R3OT/dtjgMnL9rSVb1xhXZH9cZtbdHvqnq2/oejZ3LxvxWZ+O899mmesVHeDzgr+7w5okpCP7K2MoivU7VC9XmEhQQZNlm4YtRXMLhzY1tAt+rZIgZLd2Wjc5xvbdyuuBs9a3VdixinLBwr/Xdekmmf7+wUi3o1ItCjeR275r16NSJKXQlgQCeiCmXti32Qp5veWD9/UHkxCuZ3JzbGtNQDthp2SYlImQ0qZEAnogqlJNPkloeJd7bD/ddeidYNfG+OKi8M6ERELkQ5zAhqNC1DRXJZzbZIROQNawdy13hzpxf2FQM6EZGDx7VpsDv48ASuioBNLkREDgYnNoZSwF2JlSvziQGdiMhBcJDYnuZVmbDJhYgoQDCgExEFCAZ0IqIAwYBORBQgGNCJiAIEAzoRUYBgQCciChAM6EREAYIBnYgoQDCgExEFCAZ0IqIAwYBORBQgGNCJiAIEAzoRUYBgQCciChAM6EREAYIBnYgoQDCgExEFCAZ0IqIAwYBORBQgGNCJiAIEAzoRUYDwKqCLSLKI7BSRDBFJMVj/rIhsF5EtIrJQRK70f1GJiMgdjwFdRIIBTAbQH0BrAENFpLXDZhsBJCql2gGYDuBNfxeUiIjc86aG3gVAhlJqr1IqH8BUAIP0GyilFimlLmhvVwOI9W8xiYjIE28CeiMAB3Tvs7RlrowAMMdohYiMFJFUEUnNzs72vpREROSRXztFRWQYgEQAbxmtV0p9opRKVEolxsTE+PPQRESXvRAvtjkIoLHufay2zI6I9AXwEoDrlFJ5/ikeERF5y5sa+joAzUUkXkTCAAwBMFO/gYhcDeBjALcopY75v5hEROSJx4CulCoE8DiAeQDSAfyglNomImNE5BZts7cAVAPwo4hsEpGZLnZHRERlxJsmFyilZgOY7bDsVd3rvn4uFxER+YgjRYmIAgQDOhFRgGBAJyIKEAzoREQBggGdiChAMKATEQUIBnQiogDBgE5EFCAY0ImIAgQDOhFRgGBAJyIKEAzoREQBggGdiChAMKATEQUIBnQiogDBgE5EFCAY0ImIAgQDOhFRgGBAJyIKEAzoREQBggGdiChAMKATEQUIBnQiogDBgE5EFCAY0ImIAgQDOhFRgGBAJyIKEAzoREQBggGdiChAMKATEQUIBnQiogDhVUAXkWQR2SkiGSKSYrA+XESmaevXiEicvwtKRETueQzoIhIMYDKA/gBaAxgqIq0dNhsB4JRSqhmAdwBM9HdBiYjIPW9q6F0AZCil9iql8gFMBTDIYZtBAL7UXk8H0EdExH/FJCIiT0K82KYRgAO691kAurraRilVKCI5AKIBHNdvJCIjAYzU3uaJSFpJCh0A6sDhu7lMXK7nDfDcL8dzL6vzvtLVCm8Cut8opT4B8AkAiEiqUiqxPI9fUVyu5365njfAc78cz92M8/amyeUggMa697HaMsNtRCQEQE0AJ/xRQCIi8o43AX0dgOYiEi8iYQCGAJjpsM1MAA9or+8E8KdSSvmvmERE5InHJhetTfxxAPMABAOYopTaJiJjAKQqpWYC+BzA1yKSAeAkLEHfk09KUe7K7nI998v1vAGe++Wo3M9bWJEmIgoMHClKRBQgGNCJiAKEKQHd01QClZGIZIrIVhHZJCKp2rLaIjJfRHZr/0Zpy0VE3tPOf4uIdNTt5wFt+90i8oCr45lJRKaIyDH9OAJ/nquIdNK+ywztsxVikJqL835NRA5qv/smERmgWzdKO4edItJPt9zw719LPFijLZ+mJSFUCCLSWEQWich2EdkmIk9pywP6d3dz3hXzd1dKlet/sHSs7gHQBEAYgM0AWpd3OcrgvDIB1HFY9iaAFO11CoCJ2usBAOYAEABJANZoy2sD2Kv9G6W9jjL73AzOtSeAjgDSyuJcAazVthXts/3NPmc35/0agH8abNta+9sOBxCv/c0Hu/v7B/ADgCHa648APGr2OevOpwGAjtrr6gB2aecY0L+7m/OukL+7GTV0b6YSCBT6KRG+BHCrbvlXymI1gFoi0gBAPwDzlVInlVKnAMwHkFzehfZEKbUUlmwmPb+cq7auhlJqtbL8hX+l25epXJy3K4MATFVK5Sml9gHIgOVv3/DvX6uN9oZl6gzA/js0nVLqsFJqg/b6LIB0WEaIB/Tv7ua8XTH1dzcjoBtNJeDuC6osFIA/RGS9WKY4AIB6SqnD2usjAOppr119B5X5u/HXuTbSXjsur8ge15oVplibHOD7eUcDOK2UKnRYXuGIZTbVqwGswWX0uzucN1ABf3d2ivpPd6VUR1hmpXxMRHrqV2q1jssiR/RyOlcAHwJoCqADgMMA/mNuccqWiFQD8BOAp5VSZ/TrAvl3NzjvCvm7mxHQvZlKoNJRSh3U/j0G4BdYbrGOareS0P49pm3u6juozN+Nv871oPbacXmFpJQ6qpS6pJQqAvApLL874Pt5n4ClWSLEYXmFISKhsAS1b5VSP2uLA/53Nzrvivq7mxHQvZlKoFIRkaoiUt36GsCNANJgPyXCAwBmaK9nArhfywRIApCj3bbOA3CjiERpt3A3assqA7+cq7bujIgkae2L9+v2VeFYg5nmNlh+d8By3kPE8vCXeADNYen0M/z712q3i2CZOgOw/w5Np/0WnwNIV0pN0q0K6N/d1XlX2N+9PHuMdT3BA2DpLd4D4CUzyuDn82kCS6/1ZgDbrOcES/vYQgC7ASwAUFtbLrA8NGQPgK0AEnX7Gg5LR0oGgIfMPjcX5/s9LLeZBbC0+Y3w57kCSNT+B9kD4L/QRjSb/Z+L8/5aO68tsPzP3EC3/UvaOeyELmPD1d+/9ne0Vvs+fgQQbvY568rWHZbmlC0ANmn/DQj0393NeVfI351D/4mIAgQ7RYmIAgQDOhFRgGBAJyIKEAzoREQBggGdiChAMKATEQUIBnQiogDx/6fX/5WwYanIAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "learner.fit_one_cycle(1,max_lr=2e-05,moms=(0.8,0.7)) #0.847289\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 354
    },
    "id": "KGn6QgmEEO9t",
    "outputId": "967bc52a-2546-4546-a308-c7fe5a831e4e"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>error_rate</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>0.446778</td>\n",
       "      <td>0.416921</td>\n",
       "      <td>0.842889</td>\n",
       "      <td>0.157111</td>\n",
       "      <td>1:19:15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>0.364510</td>\n",
       "      <td>0.405099</td>\n",
       "      <td>0.848756</td>\n",
       "      <td>0.151244</td>\n",
       "      <td>1:22:08</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD4CAYAAAD8Zh1EAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3dd3wUZf7A8c+TZDchEAi9BQi9SO8KAqJ4lBOxnVjurIfn6dnO87Aryol6553+Dnu7s6HiqShNUQRFkKL0LkQJCKEFQkl/fn9syezuzO5s2GSTyff9evFid3Z2dibZfGfmeb7P91Faa4QQQlR/CfHeASGEELEhAV0IIRxCAroQQjiEBHQhhHAICehCCOEQSfH6YHftdN2za4d4fbwQQlRLq1atOqC1bmz2WtwCeq0GzVi5cmW8Pl4IIaolpdRPVq/FrclFI/nvQggRS9KGLoQQDiEBXQghHCJubehCCBGtoqIisrOzyc/Pj/euVLiUlBQyMjJwuVy23yMBXQhRbWRnZ5OWlkZmZiZKqXjvToXRWnPw4EGys7Np27at7fdJk4sQotrIz8+nYcOGjg7mAEopGjZsGPWdiAR0IUS14vRg7lOe45SALoQQDmEroCulRiultiiltiulJpu8frVSar9SarX33/Wx31UhhIiv3Nxcnn322ajfN3bsWHJzcytgjwJFDOhKqURgOjAG6AZcppTqZrLqu1rr3t5/L8d4P4UQIu6sAnpxcXHY982ZM4f09PSK2i0/O1kuA4HtWusdAEqpGcD5wMaK3DEhhKhqJk+ezI8//kjv3r1xuVykpKRQv359Nm/ezNatW5kwYQK7du0iPz+fW2+9lUmTJgGQmZnJypUrOXbsGGPGjGHo0KF8++23tGzZko8//phatWrFZP/sBPSWwC7D82xgkMl6FymlhgFbgdu11rtM1hFCiJh4+JMNbNxzNKbb7NaiLg+ed5rl69OmTWP9+vWsXr2ar776inHjxrF+/Xp/auGrr75KgwYNOHnyJAMGDOCiiy6iYcOGAdvYtm0b77zzDi+99BK/+c1v+OCDD7jyyitjsv+x6hT9BMjUWvcEPgf+Y7aSUmqSUmqlUmplcUlJjD5aCCHiY+DAgQF54s888wy9evVi8ODB7Nq1i23btoW8p23btvTu3RuAfv36kZWVFbP9sXOFvhtoZXie4V3mp7U+aHj6MvCE2Ya01i8CLwKkteos1bmEEOUW7kq6stSuXdv/+KuvvmLBggUsXbqU1NRURowYYZpHnpyc7H+cmJjIyZMnY7Y/dq7QVwAdlVJtlVJuYCIwy7iCUqq54el4YFPErUo4F0JUM2lpaeTl5Zm+duTIEerXr09qaiqbN29m2bJllbx3Nq7QtdbFSqmbgflAIvCq1nqDUmoKsFJrPQu4RSk1HigGDgFXV+A+CyFEXDRs2JAhQ4bQvXt3atWqRdOmTf2vjR49mueff56uXbvSuXNnBg8eXOn7p7SOz6VyWkZnnZe9JS6fLYSonjZt2kTXrl3jvRuVxux4lVKrtNb9zdaXkaJCGDy/6EcyJ88mv0g67UX1IwE9jo4VFLM/ryDeu3HKCotLyTpwPKbbLC3VDJi6gPdWVm7266vf7AQg90RRpX6uELEgAT2Ozn1qEQOmLoj3bpyyez9cx4i/f0XuiUL/su05eRSXlJZ7m4UlpezPK+CumWv9y46cLCJz8mxmrso+pf0NJ8FbEKk0Tk2RQpwKCehxtOeIM4r0L962H4D8Ik8AzzpwnHOeWswT88vfR7J0x8GQZbsOnQDgoVkbyr3dSPYe9fxOJJyL6kgCurBl16ETZE6ezYqsQwBs+uUomZNns373EUpKPeEvJy+fA8cKOOS9Ul++85D//TlH81m9y35xohMFoW3Y2Yc9+brHCsLXzQi2cHOOf7/tKi2VkC6qn7gFdPlzqV6WbD8AwPveNu2PVnvGln28erc/oI//9xL6P7qApARPs8VxQ+A9+6lFTJi+xPbnFZeGNtf4thtOSakOCcbXvL6CS55fCsA/P99K5uTZRMrukhYXUR3JFXoVtTY7N+5XiZ9t2Msby34Cyk7ABcWeQPvCoh0AvPT1Tg4HdSD+8LPnSnxbzjH/srz86K6qE02Ctysp8te1/T1zGPvM15avP/2FZyi21p6r8OMWV/vShi5ioU6dOgDs2bOHiy++2HSdESNGsHLlyph8ngT0KmjVT4cY/+8lPLfox0r5vMPHCzmaH5rVMemNVdz/0XoAf0fkx6v3RNyesUlkd275hjUnGGZrufb1FQC4E+19XTfvNR/JZ6SBx+dt5rQH5/uD+g8/H/a/LgFdxFKLFi2YOXNmhX+OBPQq4tVvdnLz298DsDvX0zG3bMdBThRGd2VbHn0e+ZyeD33G82FOIKt+Omz5WjBj08iQaV8GvPbk/M1c/Ny3Act++8p3dH9wvuX2vtycw+pdubgSwze5HD5eGPZ1o5JSzf9+8DQb+U5AFzxbtl8SzoWZyZMnM336dP/zhx56iEcffZSzzz6bvn370qNHDz7++OOQ92VlZdG9e3cATp48ycSJE+natSsXXHBBTGu52CnOJSrYrkMnmPKpp7z8vy8HXzz8etsBRv59EcvuOfuUPyMnL5/0Wm7cYZotps3dzB+Gtw+7ndGnNYv4WWbNJT7TF3pOGvPW7+XIyUIuHdCar7d52ufX7z5C95b1TN83YfoSPrjxjLCf2+eRzyPum0+n++b6H+eeKKJp3ZSA1+M1glpEYe5k2Lsuttts1gPGTLN8+dJLL+W2227jpptuAuC9995j/vz53HLLLdStW5cDBw4wePBgxo8fbzkn6HPPPUdqaiqbNm1i7dq19O3bN2a7LwE9DqbN3UxBcVkWx7z1ewNeNzY3+NLoTsU32w5w5SvfcU7XJrx81QDA05k5pEMjGtVJjvDuQEkRrpIhfED3+cObqwC4dEBr/7L9x8oGWTVOC92vcJstLDbPeV+wcV/Effnh58N0bpYWsEziuTDTp08fcnJy2LNnD/v376d+/fo0a9aM22+/ncWLF5OQkMDu3bvZt28fzZqZX/wsXryYW265BYCePXvSs2fPmO2fBPQg17y2nOOFJbx3w+kV9hnBTRtT5wQWp4z1nOZXvvIdAAs25XCsoJj8ohJunbGa3q3S+eimIVFty6oT0SjJZlt3MONx13IlhrxuPFForflhVy59WqWjlGLq7NAJtIpLSrn+v5E7m3JMRutK1mI1EOZKuiJdcsklzJw5k71793LppZfy1ltvsX//flatWoXL5SIzM9O0bG5lqNFt6McKivn+58C24YVb9gfkTwfTWlf47bjVrVokpaWR923bvjz/1exei4FNby77iczJs01f2/hL5BliXDau0H3e+u4n/+OfvQOHwPwK2WU4UbzyzU4ufPZbf/pk8H7l5OWz02Y5gqc+3xpyhW/VKbr3SH5U/QnCeS699FJmzJjBzJkzueSSSzhy5AhNmjTB5XKxcOFCfvrpp7DvHzZsGG+//TYA69evZ+3atWHXj0aNDug3vfU9Fz77LXkmGR5WOt03N6p8ap8b31zFE/M221o3ingYoN09c/xX41Y0ZR1+VueN+7yZLf73GIJbUUnkk1nwdu94b7Xluvd+WPZZbywt+0MwC6jGZY/O9tzV3P7uGiCwmQrggY82RHViDL7zsDovnv2Pr7goqFNX1CynnXYaeXl5tGzZkubNm3PFFVewcuVKevTowX//+1+6dOkS9v033ngjx44do2vXrjzwwAP069cvZvtWo5tcNuw5AsDJohLSUlws3JwT8HppqebA8QKapJV1mBWVaNZkH4n6s+Z628nvGh3+lw2wJyjV79DxQvo+8jkL7xxB20a1Wb0rl8837uXOczuHBK0l20OHzFv5xeQK/bYZP4Qsa3v3HP/jAhtVCJOTAptL/vf9bos1Axnz1s0Ceribj++C7qqKTdpMpi/cbvn+l7/ZEfDc6gr9eKFUYRSwbl1ZZ2yjRo1YunSp6XrHjnm+05mZmaxf77l4qVWrFjNmzKiQ/arRV+i+NlnfSMfXvs0KeP1fX2xj4NQv+OVI+LSi7MMnbFcbtNNJ16ZR7YDn5zy1CIDR/1rMxj1HmTB9CdMX/sg9H64LCf6RHIswwOejCHnmdgKanY5TM43quLni5WVs25dnmjYYTW74muxcXv92Z8CyJ8PUlvFl3wR/VkFxCW8szYr7IC8h7IhfQK+kv4/H5mxi1hrzILXvqKczzFdUavHW/QGvf7nZE3zNStwePFbgb4Me+vhCRvz9q5B13luxi882BGaw2OmkC24+OGTIr968t6yt+J3lu7jg2eiaf3736nL+8MaqqN4TrEeYnHEI/TnadeBYIUu2H2TKpxtN+wImf2A/RW1/XgFvLvu5XPsB+GeTf3rBNu7/eIM/Z12IqqxKXaEv23GQz21cwUbjhcU7uOWd0GYEI2NnnJEvpgQHWIB+jy5g8GNfhN3uXR+sZVI5gucbS7NMlxeYpObtO1rAsYLigE6915bsDNs5um53WZOR1WeFkxch08WsuSNaZrtvp0M2Vu750HPyePYrz5X7x6urV0B/67ufyMlzRjXPYDVljEB5jjOOxblCd3bii8v4vY0r2Fhp0zAVgFb1a5m+Hqu77Ghmvykt1WGzbMz6+f67NIvZ68ruQh7+ZCMb9hxl3vq9pkP6je7/OPalaIPz6qN19GQRd76/xvb6p1J33Urw7943+KkqmbvuFzaZnOR2HTrBvR+ut30n9vi8zby5LHxmRlWRkpLCwYMHHR/UtdYcPHiQlJSUyCsbOLJTtLRUk2AjVcR35W311fB9acqZRej38Cf2g+amvUc5GqadW5lkqT8xL7Rt+Nf/943tz4y1E6fYcRhtp/MVL4fP7KkIpaWaKZ9uZHfuSV76nWd6x5e/3oFSiuuGtq2UfbjxLU+piKxp4wKWF3lPcN//bK9c8XPeu5ArB7eJ4d5VjIyMDLKzs9m/v3zNetVJSkoKGRkZUb3HcQH949W7uXXGap65rA/je7UIu64vNB4vKOaVbwI70PKLSvx/GArFN9sOMLhdg7DbW7R1P52bptGsXgqPGQYLvbPc/jRq456JXyCuroIzXCrD+j1HeD2oE92XSukL6Et/PEhRSSnDOjUOu62VWYc4XljC8Ajr2eXk/luXy0XbtpVzwqyOqnRALynVtoaRG906w5PzfMs7P0QM6D53zVwbUqGvy/3z/I+//fEAj87exO3ndAq7nateXQ54rpheWLwj7Lrldap3CyI2xv87cmf0ZS8tA2D+bcNCSgsYXeyt1R58pV1e0f7NVKYBUxewP68g7LEWFJeEpL4Ke6pEp+iuQycCOnCOFxSzelcu7e+Zw0OzNpA5ebY/Z7y8Zq7K5r0VQVfK3u991sHwKYe+2W5+3H8s7HqVwZd9IaoPY2aSUebk2ZYjck9FYtBZf+HmnJj3M8zfsJd15RiPEWlS9Jmrsul837yYTzpe2XbnnmR7TuXHiyoR0M98YiEDp5ZljKzNPsLynZ4BMr7bWrOJgT/bsJctYWpfG4e23/n+Gu76wHyIbXGE0Y/zN3gyb77eFv92u4q68hehjEFlcLsGHDlpf0RxRbv3Q+sUTmM8X7R1P9e8voJnvrQeVOWzO/ekv5kxkhveWMV5/45N86DWmteW7OTAsQLmrf8FgK37Ite0r8qGTPvSP36kMsUtoBeXatOCSgCuRBXS+deucR2+2pITMJpz0hur+NW/FrPzwHGWbD8Q0vP92NzAolfBfHNU2k2zC56Zx8p5ceyQFLFjHFswsksTej38Wbm289iczSFXnFbVIe166zt7OfaHjnuuiH+OcBcKniD0YIQJuL/cvC9s3fxgW/bmkTl5NnPX/WK5zracYzz8yUZufvv7kFThGct/jqrOvY/WOqCiaXWxee9RLntxWVSZcUZxvUJ/6eudpst35540bSu++rUVXOOdvcborL9/xRUvfxdyFR9pdp1T/aMCTG+ZjXnewhmOm0xabdfeo/khmTj7TMoiZx8OHA9RUFzCS4t3nFJziS8w+m5Ci0tKwzYFvB3hRHHt6yuZNtdeTSIomwXKl5FjxndXcORkcUCdoa378pj8v3Xc+q51LSArz371I53vm0fuiehPBvH00KwNLN1xkO+9BeCKSkrJnDyb15eYx8pgVaLJJdgXm3JCF9rIO91lMUBIiFMVTdkBs3IMu3NP8tZ3P3HSm9Jptrmhjy8MeP7ioh1MnbOJd5aXf8RrWXkLT9B8Yv4WznlqET8fjP5v5XaLwFpQXOIvn2H1+XYY77C1hg+9o3PLM/LYV+ahvFMgxkq0zXTLdnj664q8P09fCvBDn5i3ZgSrkgFdE1pCdnlW5JKlS3fYL0wlhJkGHGVEwg+ckbCefmoLPdQOOquf2f3jelpwgEYcIY0TJFOI1QiGM4Km3fO598P1PD5vM6t+OsT2/ZHbiHO9wcBXPyf3RGHUUxL6x1p4d/VFbx/M4XJcuX5oUf6g833zuOEN8wGBdgK6sXnVt3qp1qyIQTqqr1m1ohSVlIYd5FSeyqxQNlgu2qy2Kpm2uC47l9Na1A1Y9olFPRajFTaCPnhGMprd8grRI2Enr7ufDH1hH2AyaK9AJ8HfaqGTkvk2uZQC7aIQF0efnspMdxGFOokCPMsKSSJlXSpblkMBLiYneZYVaDeFJFGIi9KV+yHRTUmim5+/3cDwBBdfzd/IH9oN54pnV5CeVoe3/nAmTTnkfw8lRZCQ5P/rN8YXY4A0tuNv2ZvHhGeX8O3kkSHHdOh4IQ1qu239vHzBbIHZXTWQZeNOwBe0tC7bzr0frY+YEWOl/6ML/I/DTbloJfvwCbblHOOszk3CrldUUkrHe+dy/dC23Pfrbqbr7DxwnNJSzdWvr6Bb87pMHhO52iqUv3xGlQzo7qQElmyvuKHWvunPhAj2fWlHJhRMwU0RblVMMoW4KSaZItyqyPu4kGSKSVZFuCnihj4ZbNt9gDVHc3CrIpIpIvtAMcmAWxWTSr7n/RSTWlREUqLnfb5lCcrwx/vpfwDPrfNLxpj6yqPMTgYKgWfgO+PJ5ZHrAAVJKZDkpkWCm2+SNQXaRaPP6vKRu4jCbS52bXXxistzEkldUJvHEzVH3v+I+5PyPCcWXBRqF8cXbaRBo3RISoakFH4+UkyLRukMTVgXcoJ657NvaMxhCnBB4QlISqaUBNZk59KndX0+MMlOC+YP6IY7nvIGc4ADhqkMk23MnjVw6gJOb9+Qpyf2AWDk3xdRWFLKzsfGhq2p78uOe/O7nywDOnhKmizPOsTirftDAvq89Xvp3CyNNg1SA0a3+zqCH4rQSR0s7gHdLLd7675jHDhmfUt4+7urefj80ypyt0Q1NKF3C77YnENehBLB4eSRymrdwfPE5kXSnoI2vLG9vLVQNC5KPCcQihjTpT7fbNnjD/jJFJGsinjnmr78/rUlJFPEXaPa8tyCDbgpxk0R957bDkoKoNjz7/vtv7Ar7zBuihhQuza5hw/gpohUVUA6x3BTTL3iEjomFlB/31paJuZ7Pk95f27LA/fQN+vrm2YX7Uvhct/J5W/eI1KJdCpNoigllQ8LFPnuRApww/OP8T/3SQpxkfvCc6TXTYNEN82LEngs6TCFB5MoSPLerWiX98SRRAFuji7dTxEuGqbXhcRkSHJ7TmCJ3v+Tkv2P0zjhf+/r32bx6pIsXr6qv+VvICevgI9X7/EH9EJvc8eirfsZEeYqXfnvfsL8eoHlWdZNR8aLy21Tx/gf1/feIdmdS8An7gF9zNNfmy4/FCZV6cMfdlu250VSlXKJRWwdOlHE/13Wh6tfC82EqkhvnFJhK0URSRSRxHFq8fbmEqBp4Coa6HgOn5d6rjw/nQ9QNgHxvcMDR10++fy3rCjyND/W/SWJo0UmJzjvn9dD53bzd7gpSnFTzOu/7c3pbWqzNmsfew4e4Z/z1tG3RSrb9hzArYoNJ5tiklWh/8RyZf+mtK/v5rPVWWQfyGVUy3SWbd3jf0+7tPoc07tJVkUUnzwCOheKC6hVkM/IxKP+7SRTRJIKyuwJX7E5wDrD3UvBj547Cp6o7b+D8ZwQkv0ngddcRynERf4775OSUou/Je2lABctVyyA3Y0M70kJOJEk4GJYwjpKccOuht7tebbraxIrwHOCKiaRSLMFG8fDlHesr62ArpQaDTwNJAIva61NZ2dVSl0EzAQGaK1tlU2MRepgNMqbSyxCvfS7/pVaHTOSxVv3899rB1bItru3rMv63fEbpRuuFG7uiULSUz1XdDlH8wP6ksIVeoPAq0tNAgW4yU+qzbd7E7j8TV92TWu27AZoFPau5bXvPOULZmat4Iu9OUwNSswYo5sxt8hbiXMvfHTTED5Zs4cz2jfkuv8Efo8S/Xct3iDvbcr64tbT/XciFOdDSWHZc+9dyiMf/xDQJOammOu6NYfiQijOJ+fwUeola5J1IZQU0kDl4aaIxH2HQBcxKvEoboqovaMEtlt3fruB//ruWl4JfO27oP6WEq08/R3TanlPCJ67jLnuQgq8gd/99gu84jpKAS66L2sC29N5JGkvhXjuVli41nPSCCNiQFdKJQLTgVFANrBCKTVLa70xaL004Fag8kvfiZi5bGArWtSrxT8+3xpx3c5NreuTADw6oXvI/KRGl/ZvxbsrPeUYGqclm7abXjMkk9eWZJm+/1SDbK9W6azZZa8iYXote52EFcU4kjpY7ymfs+6hc3lw1oaob9GnfBqaDldcorn29fL/GRdZtEHMDSqr7MsACS6MB1BCIidJxJ+j4ttksx6ep1qTV1BM3RSX/z0PzdrA1n15fFsSWsPpul977mIKiksYeN88OjdNY/7tw8jLL+L8hzwXecuvO5smaSkMMIwtyXpsrKfTuaTAf0LwnThO5p/giucWk5pQzJtX9/Euz4fiQu5+f0XAHYevb+WGnhlQUsCeA7nUc2t+ztnt7UspgqITNFGHPU1ih/bCkRLGGu5cWDQr4s/ezhX6QGC71noHgFJqBnA+EPxNeAR4HPiLjW0Kgwv7tGTJjwf8MyjFy8guTXjswp4AtgJ6QoT+pisHt7EM6FnTxnH4eKE/oN/1q878ZWZoaYY+retbBvRZNw2l3T1zTF8z6ts63bSUrAL6tE7nBxtlZss7cq+yDJi6wD/z1qn641ur7Az7sGRn3tlT9fyiHTw+bzN1U5JYfNdZpKe6Q6pfmvEd107vyNkf95dl/uzYfzxg/mDwpIzWTnZ7mlqCLo5LC4r5Xu8loRToeE7Aa+/MCMzS87lhrOfEcsbk2bgSVcCk62uuOJfzpnhOLv93bh/O69WCfoaTy8I/D6OuqxQeTrc8Pjs5PS0BY1WrbO8yP6VUX6CV1jpspSGl1CSl1EqlVNW5T4+zv1/Si6cu7U3TutEVsrey/N6zy/3eThGuuGu5Aivg+XKMoy3u53tffUNqnFW+criSxQkJirSU0GuSnhn1Ap6/HtQM08Vb+VApuHaIvVKsK3+ylxIbL7EK5kBAkIlWaamulHLGj8/zjFY9ml9sqwxCXvBEL95DNH7rCotL2RlUomH4k18x6G8LTGtG6aD/oxX8c175U9nPzezv4ax/LGbYU+aTUfuc8sAipVQC8BTw50jraq1f1Fr311pbdznXML7f2479sakuF3yFEQ3fDE5W7vxV54DniUrx0u/6s/DOEVF9TqbJ51hlhzWuE77N0CyjpU5yYJA33pbfOKI9U87vDngG3fy6Z/NIuyuiUNHF447YrKcUzFe6wTfit9Bk4M7vXl3OjUEpzQeOFbDvaAH/NLlj9eXgaw2vfrPTViXUl8L8fG4wzDBVy21ePjjSJO12AvpuoJXheYZ3mU8a0B34SimVBQwGZimlJGjb4BvJ9/D4+KZh/u+PZzBxQNmv+fv7R9G9ZeBtY8cmdQKeJyQoRnVrSpuGtZl2YQ9bn6MUfPqnM0OXW/TrK6VIS44uGevqMzJDlr38u/68ff0g/jq6LA9Yebc/qG3gXUCTtPAnEWHNd+VcUeauty7yFY4vKy64eT/4uVUJg3kbQqdVNK455dONjH3GPGPPaOqcTZbz0xoHE13z2opyjcWxE9BXAB2VUm2VUm5gIuBvnddaH9FaN9JaZ2qtM4FlwHi7WS41ne8KoVGUQeTqMzJtT+BhR9/W9QMGUTSo7WbqhMAgHVzX21h3e+LA1lw+qDWR7HxsnOnVR7ghzsM6RzeTz08moxPP6daUMzo0AkKnFnz3htMD1k2qwhNE1HRm5X0XbNpnGYh9Thb5auiUrXeisDhk2H64mj3Bn221aqQ6Ob5JeCL5q0W573AiBnStdTFwM55M0E3Ae1rrDUqpKUqp8VF/okP0bW3dMVEZLujTkmcu6xN2nRtHtD+lz+jVKj1gZpmEoKgbPG9r24a1/Y+jTR/0pd2ZMQuwrRqYT+wNcOD4qXUuO3kKt+ruXwu2hSz74efciCV9n/liG68v2cme3LL0zzOmfUmqO/DuLydMYkLHe+f6O8dz8vL53/eho2CPnCxi2JMLQ5aXR3nq0NhqQ9daz9Fad9Jat9daT/Uue0BrHZJHo7UeUROuzh8e3z0m2/ENdIp2FnM7RY9SXYk89ZteAIyLQVtx7aCmj+SgOhmdDNOs+ebRvHxQa/40skPEbQ/r2IhbLNYLnoEH4Pqh7Sy3dW63ppavGVk18+hyd3OJinbQYsChneD30CcbA8oY554oCigTAJBXED5vPy+/mN88v5SBU7/gYZMKiOVt44+VKlltsTpoWd/6CjEaa3Z5aqd3a2Ge5mQlKTEwGI3q1pQBmfUDlpVquLBvBlnTxjH98r4h2xiYGX7S62AprrKvy5d/Hk5KUNaL21s3o5khY+dvF/Tgz+cGdqaaUUpxx7md6d0q9M4n+E5g29Qx/O506xnqjZ2gVp8F5oH70z8NPaWUPREfdi+IggP4rTN+iOpzFm7JCTuU/4n5FduHEEnch/5XV7GaiNcXVIKzUx6Z0J2UpATSUlymxcR8k+hePqg1g9o24PzeLUPWCW4PvHdsV6bOKZvFqUdGvbBfzmAphol72zWuE/J6x6aeZcHZMOHMunkIS7aXlT0+r1cLVgcN9gmu/OeKUHCpWUKaUh8AABnvSURBVD3Pz/Kivhmc1yv0ziRcDY7uLeuFLhRVyiiTqd1mBM8XbFO4mlFm7jIZK2G0aEt8p6mUK/RySkxQvBfUmRbOBzeeYf6CxYXFbwe34ZL+rRjdvZlpW3nbRp726r9d0MM0mEPoVctZXQI7F0Nycy0svXskV53ehsHtGoZdr1GdZLKmjePifhkhr/1xRHveuC60Xb1nRnpAW/+1QzJD1rljVCfuG9fV9DNn3zI0ZFlaiousaeP4x296mRZX8p2Kra7qcmNY7ydcW78on23lmHy5svq5T8R5AJoE9HJKTkpgYNvwTRa9DM0H/drUN13Hzo3i+F4tuLR/WUrhlYMjZ5NA6BVocPwa19NelkzzerV4+PzuIc080bhrdBfO7Bg5W0UpxcYpvwpYluJK5PozzdvMyzMgq6xcq7lY1hdadOdZlq89eF63iN8hERuV1dEdKdumokmTSzkYMz+szL5lKI3qJDPob6E1OGq7E/0DBOx+AaZe0N0/TD7TkE0STvAVePAnBU8iEkmsmpkiCc488Jl6QXdqB71mzIBJtRiMESp0Ioho1Kvlsl21M7j936hnRj3SUlwsr4SRlaJmkCv0INEGuTvP7WSxnXqW+cx3jy1rPjCLKU9P7B2yLMnQbmw3EJ0IGlUW/D5fGmLzevaucoPTFivbFYPaMKFPYPNSvVou/jDc02Rjd97P2smewG/3uIP5MnhOlTsxMSRTSIhT4chv09vXDyr3e8/vHd1gnZtHdgy5YvfVCkmyqF512cCyJhOXSTOGVZt4tILbj4MDXmFxKUvvHsn824fZ2l68A7oZpRS3j+oIQNtGoR21Zro0q8vTE3vz+MU9/cv+c+1A27nzpTbvqnwpo40tBo25kxIqvXy0cDZHBnTfiEA7RgSNQrygTwZTyjEbUsv0ss4vX+dhokWbs3Fpq/pldU0m9G5BozC1S3w1SuzmSQfnnpsF9Ob1akVM8/OpqgMok5MSee3qAaadrlbO790y4LiHd2ps+8q7R4a9TJgL+3o6hxf/xbwd3ZWoWLilbC7OUTbz54Ww4siADjD/NntXnY9OCBwgpNH87vTMqD9v6gWhA42Cm1zO7Og50SQkKH/WRgdDfZR/TezDyvsCy3AaXTawleVrdgR3IEZ7wV1ZbejlcVaXJmFPhrHUqWkdtk8d478Ti8Sq0JIrMYGDhrS5v462n+5ZnVThr43jVImAbpyZ+xXD3H9DTa60HwgzGatR52ZpvHX9IJ6/0jOgpplFNkRws6vV6MFIRnRuErJv7qB86VevHsD6hz0ZHNef2Y45t5zJeVHUY1Ems7qb+e3gNqY/p0Z1ktnwcFkGiVVTQKTPr+lKSz19GuGaoNY8cG7A8/9cOzDkBOBOSqB5etn3cntObCpuVjVSSqHyVImAnmII6IPaNWTxX86iS7M0nv9tv5B1z+kaeFvatbl1J+aQDo3o18aTFmZW1AdCg2PD2uY1RYJLsprxXYH7SrIGZzi4EhMCthPt6NCbRnTg4n4ZXDnYepQkeAYlXTvUvM537eQkvrvnbP59eZ+QkZ7CY8Edw5l+eV+W3j2SzY+MDnnd13R1s0WpgtvP6US91MBmrOGdGtMxqN68KzGB4YZmHqcWBWvX2F5WlpN9cOPpldKkViUCuvFWXgGtG6Yy77ZhpkG0taGW9qC2DagddDvrDsoa8F0lF1oE9IZ1AgO4VZpZ8HbNdGyaRta0cfSPcki9XfVSXfz9kl4hNVWi1bRuCr+2mYNuZmyPZpFXOkU/3D+K1Q+MqvDPMdOhSR3G9WxO83q1TE96vmsAq4FWVn0cvYLa3hMTlP+C5MyOjahf215fhk/TutWjzG+sav2XR3knfLnO4oKovPq1acDZXUIHucValQvoVrnExk5Hn2cu6xNy2xucNeBK8rxudoXuTkygdnISjbxB3fjZwVcV5R0wsPqBUax96NzIK1YTWdPG8ewVoXdOsVa/tjtsBcZ48o0wrVcrcOIMX3C2ahK7ZkjbgMk9khIUnZqm8eZ1g3jpd/1tZ+n4PHdlxf8eYmFkJQQyn+SkhIA7nfJO+BLPej7/vLRXud9bJQJ6wKQDFu2SX945POT2t14tV8SOvVquRCYOaMVb1w8KWdd31b7yvlFkTRvHxill2//fjWcw99ayiRje/n35UiHTU922s0hE9eA7tycmKL6+6yxaN0jlqtMz+dVpnltqq5ICiQmKeYbOel9NmqEdG5HiSrTMo+/U1DzQ921dP+xsUb4TjvHEA6F3CoC/Q/mOUebjKk5FZU4YsuXRMbx2zQDA/CLQroqouGm3Cyo4eeH0CCU3jOIe0Mf3ahG2tOs7vx/Ml38eTnJSYsjtb4JSETMvlFJMu6gn/do0iKq7Mz3VHdA+f1oLKdokoHerdH9fCUCrBqksvussmtVL4eJ+GTSq4w4Z/GSU4kr0X8AEN+MZs3RuO6ej/7GxONmaB89l+uV9/Ve9vpo+ZtY8eC4//m0sV3krU3ZoUocOTerQJChgdGhSh28nj2TNg+dyy9kdzTZ1Sip7OLzdkdThVMQVut1tBmdrmWXQWYl7QP/D8PZhM0tOb98wpLKfL4grBZf0Dy0EZeWqoKnJ7Jz5xnSv+PZiUX18dNMQ0izuuDLqp7LyvlGmlSiNbhzRPmL5CONsVBf28XzHz+vVgnq1XIzr2ZxXrx5ga38TExQp3qbEi/pmsOCO4SEjZM/p2hR3UkLIlbwZq7uFcII76Ds0sd6G8YTSLuhkZbdT0dcPFu0cA1VF8J1FNAP64h7QMxulRp0P7UtnTFCKC/pksGTySFvve+DX3dg2dYz/ebIr8uFPv7wvWx8dE3E94XzGK/OK8spV/Zl765kBJ4Xh3sFv15hUojRa7s1eCnbtkLbcMrKD//1/iVDe+Lkr+vLKVf1D5loF++M7jPPABmeiffAHi8qjeEbhfvNXz0CsD28aEvCaWUkMM75wcio3BsaTQXmbW4PZiXPpqa6QVodIk7cbxS2g+3Y5Qamo07Wev7IfX/55uP/A7baVKaUCamnbuTVLSFC2MlyE85kFuFg7u2vTkADYtG4KWdPG0be1ecVOnyZ1U0wnLUlxJXLHuZ39TZZWdxg+Y3o05+yuTTm9fegdbHAfV/BVtM87kwbz51GdeCuoDMc9Y7sEpHQGX3WXak1G/VSypo0LuGPo3rKuZdE2H99YE19fxKkMhBtqqAxaN8VFeuqp94Od0d78gqCPYTrLkhIdEviNP3OrMtI+VSJS+QpPBc+4Y6WWOzHibW04t5/j6fiJlM8thFG4yokVJdrRr8Ht41aMCQZpKeaB8qK+gc2ZZiWgCwxZZcZmou4t6/GnszsyJGhw4FlB9YV8hdV8rK6q7bSetPAO0mpUJ5lUdyL3jA0f/MIxnmgSExSrHzj1TLVWDcyvtB88r6zUSHGpDtvEEmk8TNzK5/rOtr6zqJ2StHak2cjR/tPIDkzo04I2Meg8ETVHZQ/8+e+1A+nU1F55gWgZEwyCA7dP8AnM7PgLissqet7/627MWrPHdFuPTujOml25IYOrjAHqjPYNuXZoZsR9t7LjgCffPcWVGJCx1q9NfVb9dLjc2/UdtytRUVQSfTvODcOs57+FwNpOJaXadA7dz24fxpUvf8fZXcP3I8TtCj2zUSqf3Dw04nRidtX1XmW8a2MWoYQEJcFcRK2yq00O69TYP51eRTKr+An4B+357hLMrpKNV+jhfjxXDm7Dk5eE5lcbB1O9/fvBlnnjvm3/cP8o/m6yHfBM+mzmThtz2obja6Jaend0g5S+/PNw/nFJLyaP6RJ5Za8SHdjk8uZ1niarTk3TWH7vORHLdcQtoCcoZbtqXTRapFf8H4ComZxay8aqrTk91c3sW4byD28ZYLPcbGNAj+aE99WdI/jnpb1sD/xp7D2p1K/tNp3iMBx30qn93nwnVeOVs51SIO0a1+GifhkRvzdKwYV9PamuD40/LWD9oVF2xFeJNvRY8NXViNRxIkR5VdVaK1/fdRZL77aX6WUmXMA5rUU9emd4Ou187d3/Z5jjtlX9soSEaH48mY1qc0GfyIHZV9L4ikHl7+/q3Mw4nqQub143iJ2PjY16O8YT1hd/Hu5/PMFkDgWrontWKZtTzu/OfeO6coVhrgSrDudwHBP9Jg1rz6Rh7SOveArevG4QhSXxnQRWxI9VGdx4s+pssyvShXW9VFdAH9d5vVqwP6+AKZ9u5Oohbbn/o/UA5WpfBk866O7ck+b75v3fTsbK4HbmWUjGq+nZt5xpus4jE7r7j8Pn+/tHBUw1aJyvxjia8+yuTflo9R5uObsjg9s1sMxmARjYtgHbc47xm/4ZvLcyG/AcW53kpIB5c1+7egBdmkfff+KYgF4Zor39Ec5Sv4rWlgk2/7ZhtgYJ+ZTnvuOqMzJJS0niwr4Z/kBY3hGhb1xnnef919Fd2J9XwAAbKaPLdpRvbtYZkwabltduUNsdMErXqknpvF4t6NwszVYHtu+81KNlPe48tzPvr8qmm0nF2LPKWf/GMU0uQlSUBXcMZ0TnxpUysCgWOjdLs9WZmuIdWFeevoHEBMUl/VsFXDk3qO02TW08Fd1a1GXOrWdatlnfb3N+hIfHn0Z9i1zywe0akmlo3jBrQoHwfQR2s5F82yjVnhTTm87qENO+GblCFyKCDk3q8Po19qe3qy5inbWj0bx3w+mVWrvluqFtycnL54VFO8Kud9UZmSGlP6wUWex/UlA20BWDWvPZxn22tulTFtAr5mckV+hC1FCJMQou828bxh2jOpGclEhiPEZWxyg2zrrZU2rgNIuJZ4JTrKde0IMV91pPGWnGdw6tqDIzcoUuRA3lDy7mc7/Y1rlZGp1tzq9aEXxjSqJNQhrZpUlAgb6eGel8dvswOpzCKPRIRnVrymtLshhYQWUkJKALUUN1aprGyp8OB2RvVEd1vIMKx/SwLsNtxqxiZUWNzPU5o32jmI2KNyMBXYga6uWr+rM2+0jEYl1V0Vd3jvAH8qo5OiA+JKALUUOlp7r9A3eqm0yzQTfVs/x5TNkK6Eqp0cDTQCLwstZ6WtDrfwBuAkqAY8AkrfXGGO+rEEKE8PcFVEJE//CPZ5CXX1zhn1NeEQO6UioRmA6MArKBFUqpWUEB+22t9fPe9ccDTwGjQzYmhBAx5ivJkFQJnQF9ItSkjzc7V+gDge1a6x0ASqkZwPmAP6BrrY8a1q+N3PwIISrJOV2bMmlYu5Da6jWRnYDeEthleJ4NhIzVVUrdBNwBuAHTSkFKqUnAJIDWrVubrSKEEFFJSkw4pcksnCRm9yha6+la6/bAX4H7LNZ5UWvdX2vdv3Hj6tkZI4QQVZWdgL4baGV4nuFdZmUGMOFUdkoIIUT07AT0FUBHpVRbpZQbmAjMMq6glOpoeDoO2Ba7XRRCCGFHxDZ0rXWxUupmYD6etMVXtdYblFJTgJVa61nAzUqpc4Ai4DBwVUXutBBCiFC28tC11nOAOUHLHjA8vjXG+yWEECJK1byKgxBCCB8J6EII4RAS0IUQwiEkoAshhENIQBdCCIeQgC6EEA4hAV0IIRxCAroQQjiEBHQhhHAICehCCOEQEtCFEMIhJKALIYRDSEAXQgiHkIAuhBAOIQFdCCEcQgK6EEI4hAR0IYRwCAnoQgjhEBLQhRDCISSgCyGEQ0hAF0IIh5CALoQQDiEBXQghHEICuhBCOIQEdCGEcAgJ6EII4RAS0IUQwiEkoAshhENIQBdCCIeQgC6EEA4hAV0IIRxCAroQQjiErYCulBqtlNqilNqulJps8vodSqmNSqm1SqkvlFJtYr+rQgghwokY0JVSicB0YAzQDbhMKdUtaLUfgP5a657ATOCJWO+oEEKI8OxcoQ8Etmutd2itC4EZwPnGFbTWC7XWJ7xPlwEZsd1NIYQQkdgJ6C2BXYbn2d5lVq4D5pq9oJSapJRaqZRauX//fvt7KYQQIqKYdooqpa4E+gNPmr2utX5Ra91fa92/cePGsfxoIYSo8ZJsrLMbaGV4nuFdFkApdQ5wLzBca10Qm90TQghhl50r9BVAR6VUW6WUG5gIzDKuoJTqA7wAjNda58R+N4UQQkQSMaBrrYuBm4H5wCbgPa31BqXUFKXUeO9qTwJ1gPeVUquVUrMsNieEEKKC2GlyQWs9B5gTtOwBw+NzYrxfQgghoiQjRYUQwiEkoAshhENIQBdCCIeQgC6EEA4hAV0IIRxCAroQQjiEBHQhhHAICehCCOEQEtCFEMIhJKALIYRDSEAXQgiHkIAuhBAOIQFdCCEcQgK6EEI4hAR0IYRwCAnoQgjhEBLQhRDCISSgCyGEQ0hAF0IIh5CALoQQDiEBXQghHEICuhBCOIQEdCGEcAgJ6EII4RAS0IUQwiEkoAshhENIQBdCCIeQgC6EEA4hAV0IIRxCAroQQjiEBHQhhHAICehCCOEQtgK6Umq0UmqLUmq7UmqyyevDlFLfK6WKlVIXx343hRBCRBIxoCulEoHpwBigG3CZUqpb0Go/A1cDb8d6B4UQQtiTZGOdgcB2rfUOAKXUDOB8YKNvBa11lve10grYRyGEEDbYaXJpCewyPM/2LouaUmqSUmqlUmrl/v37y7MJIYQQFiq1U1Rr/aLWur/Wun/jxo0r86OFEMLx7AT03UArw/MM7zIhhBBViJ2AvgLoqJRqq5RyAxOBWRW7W0IIIaIVMaBrrYuBm4H5wCbgPa31BqXUFKXUeACl1AClVDZwCfCCUmpDRe60EEKIUHayXNBazwHmBC17wPB4BZ6mGCGEEHEiI0WFEMIhJKALIYRDSEAXQgiHkIAuhBAOIQFdCCEcQgK6EEI4hAR0IYRwCAnoQgjhEBLQhRDCISSgCyGEQ0hAF0IIh5CALoQQDiEBXQghHEICuhBCOIQEdCGEcAgJ6EII4RAS0IUQwiEkoAshhENIQBdCCIeQgC6EEA4hAV0IIRxCAroQQjiEBHQhhHAICehCCOEQEtCFEMIhJKALIYRDSEAXQgiHkIAuhBAOIQFdCCEcQgK6EEI4hAR0IYRwCAnoQgjhELYCulJqtFJqi1Jqu1JqssnryUqpd72vf6eUyoz1jgohhAgvYkBXSiUC04ExQDfgMqVUt6DVrgMOa607AP8EHo/1jgohhAjPzhX6QGC71nqH1roQmAGcH7TO+cB/vI9nAmcrpVTsdlMIIUQkSTbWaQnsMjzPBgZZraO1LlZKHQEaAgeMKymlJgGTvE8LlFLry7PTDtCIoJ9NDVFTjxvk2GvisVfUcbexesFOQI8ZrfWLwIsASqmVWuv+lfn5VUVNPfaaetwgx14Tjz0ex22nyWU30MrwPMO7zHQdpVQSUA84GIsdFEIIYY+dgL4C6KiUaquUcgMTgVlB68wCrvI+vhj4UmutY7ebQgghIonY5OJtE78ZmA8kAq9qrTcopaYAK7XWs4BXgDeUUtuBQ3iCfiQvnsJ+V3c19dhr6nGDHHtNVOnHreRCWgghnEFGigohhENIQBdCCIeIS0CPVEqgOlBKvaqUyjHm0iulGiilPldKbfP+X9+7XCmlnvEe71qlVF/De67yrr9NKXWVYXk/pdQ673ueqSoDtZRSrZRSC5VSG5VSG5RSt3qX14RjT1FKLVdKrfEe+8Pe5W29JS+2e0tguL3LLUtiKKXu9i7fopT6lWF5lf7bUEolKqV+UEp96n3u+GNXSmV5v4+rlVIrvcuq5vdda12p//B0rP4ItAPcwBqgW2XvRwyOYxjQF1hvWPYEMNn7eDLwuPfxWGAuoIDBwHfe5Q2AHd7/63sf1/e+tty7rvK+d0y8j9m7X82Bvt7HacBWPCUhasKxK6CO97EL+M67n+8BE73Lnwdu9D7+I/C89/FE4F3v427e730y0Nb795BYHf42gDuAt4FPvc8df+xAFtAoaFmV/L7H44dzOjDf8Pxu4O54/9LKeSyZBAb0LUBz7+PmwBbv4xeAy4LXAy4DXjAsf8G7rDmw2bA8YL2q9A/4GBhV044dSAW+xzNq+gCQ5F3u/37jyQw73fs4ybueCv7O+9ar6n8beMagfAGMBD71Hovjjx3zgF4lv+/xaHIxKyXQMg77URGaaq1/8T7eCzT1PrY65nDLs02WVyne2+g+eK5Ua8Sxe5scVgM5wOd4ripztdbF3lWM+xtQEgPwlcSI9mdSVfwLuAso9T5vSM04dg18ppRapTzlS6CKft8rdeh/TaK11kopx+aEKqXqAB8At2mtjxqb/Zx87FrrEqC3Uiod+BDoEuddqhRKqV8DOVrrVUqpEfHen0o2VGu9WynVBPhcKbXZ+GJV+r7H4wrdTimB6mqfUqo5gPf/HO9yq2MOtzzDZHmVoJRy4Qnmb2mt/+ddXCOO3UdrnQssxNNUkK48JS8gcH+tSmJE+zOpCoYA45VSWXgqro4EnqYGHLvWerf3/xw8J/GBVNXvexzao5LwdAi0pazz47R4t5OV81gyCWxDf5LAjpInvI/HEdhRsty7vAGwE08nSX3v4wbe14I7SsbG+3i9+6WA/wL/ClpeE469MZDufVwL+Br4NfA+gR2Df/Q+vonAjsH3vI9PI7BjcAeeTsFq8bcBjKCsU9TRxw7UBtIMj78FRlfV73u8fkhj8WRH/AjcG+9fWjmP4R3gF6AIT7vXdXjaCL8AtgELDL8whWeSkB+BdUB/w3auBbZ7/11jWN4fWO99z7/xjuqN9z9gKJ42xbXAau+/sTXk2HsCP3iPfT3wgHd5O+8f5XZvgEv2Lk/xPt/ufb2dYVv3eo9vC4ashurwt0FgQHf0sXuPb4333wbfflXV77sM/RdCCIeQkaJCCOEQEtCFEMIhJKALIYRDSEAXQgiHkIAuhBAOIQFdCCEcQgK6EEI4xP8DGAEW/NHTojkAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "learner.save('first_cycle')\n",
    "learner.freeze_to(-3)\n",
    "learner.fit_one_cycle(2,max_lr=2e-05,moms=(0.8,0.7))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 17
    },
    "id": "wf350mCfqAPc",
    "outputId": "a71d54b9-33cc-42e0-cdbe-ec373378762b"
   },
   "outputs": [
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "def get_preds_as_nparray(ds_type) -> np.ndarray:\n",
    "    \"\"\"\n",
    "    the get_preds method does not yield the elements in order by default\n",
    "    we borrow the code from the RNNLearner to resort the elements into their correct order\n",
    "    \"\"\"\n",
    "    preds = learner.get_preds(ds_type)[0].detach().cpu().numpy()\n",
    "    sampler = [i for i in databunch.dl(ds_type).sampler]\n",
    "    reverse_sampler = np.argsort(sampler)\n",
    "    return preds[reverse_sampler, :]\n",
    "\n",
    "test_preds = get_preds_as_nparray(DatasetType.Test)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 195
    },
    "id": "-bKOBN9v_qZz",
    "outputId": "b8f8eb2c-a4a9-4794-a07b-e2d49350aaba"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   label\n",
       "0      2\n",
       "1      0\n",
       "2      0\n",
       "3      0\n",
       "4      1"
      ]
     },
     "execution_count": 29,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sample_submission = pd.DataFrame(np.argmax(test_preds,axis=1),columns=['label'])\n",
    "sample_submission.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "id": "hJfYZAeWAAWl",
    "outputId": "3e917b45-569e-44c3-8511-d0204b73ed51"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(30000, 1)"
      ]
     },
     "execution_count": 30,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sample_submission.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 84
    },
    "id": "-77qKeWdADOk",
    "outputId": "06ff1a38-2c34-4d80-a6eb-0fe3e613588e"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1    13855\n",
       "0    13784\n",
       "2     2361\n",
       "Name: label, dtype: int64"
      ]
     },
     "execution_count": 31,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sample_submission.label.value_counts()  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "id": "2L_p3ddGAI3o"
   },
   "outputs": [],
   "source": [
    "sample_submission.to_csv(\"roberta_base_1epochunfreeze_2epochsfreeze_3_lr2e5.csv\",index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "4QCMBjjKsTBD"
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [],
   "machine_shape": "hm",
   "name": "MH_Multiclass_85022.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3"
  },
  "widgets": {
   "application/vnd.jupyter.widget-state+json": {
    "0dd0eb1c74eb447fbc6595dc7064405c": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "Downloading: 100%",
      "description_tooltip": null,
      "layout": "IPY_MODEL_1e243edc2d6a483c9a1307f43602bb9a",
      "max": 481,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_cb4af4030f764ecdbd007d7555384294",
      "value": 481
     }
    },
    "130f67d003784be4b1178e2847f3527e": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_717db8fb535c40319b1c1f0d348c6f0d",
      "placeholder": "​",
      "style": "IPY_MODEL_bdb04ac4a9394c72a7f74c64e8dfe44c",
      "value": " 899k/899k [00:01&lt;00:00, 535kB/s]"
     }
    },
    "1e243edc2d6a483c9a1307f43602bb9a": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "1fe38b4c12554aaea903d0116b5483a9": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_8715b7c80dd846d499d6f13f41c5b9ff",
       "IPY_MODEL_481df8579f9e47f28d8bcfa0157bf242"
      ],
      "layout": "IPY_MODEL_c2ee60b580c349cc8bb4612e159e4c34"
     }
    },
    "259d27bec617486c9c146ea77f99b488": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": "initial"
     }
    },
    "2d63cfdd601e4170afebd9ccede64520": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "476732bced504081baa9d838b4e2ce1e": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "481df8579f9e47f28d8bcfa0157bf242": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_714f8746bc2b4ca7acb02f1e46ae2649",
      "placeholder": "​",
      "style": "IPY_MODEL_a0d5af49e8534efdaf351c57f3a451ce",
      "value": " 501M/501M [00:06&lt;00:00, 79.2MB/s]"
     }
    },
    "4a5b322be1bf4da29989a67346fec70f": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": "initial"
     }
    },
    "4ad144ee7d2545efbc2b33b5c48acdc4": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "5f35e41ac97a4f85a037d970351715a6": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "Downloading: 100%",
      "description_tooltip": null,
      "layout": "IPY_MODEL_f2ef9acb3545464caefb040ced4e70c0",
      "max": 456318,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_4a5b322be1bf4da29989a67346fec70f",
      "value": 456318
     }
    },
    "6d87fa5b6f4046b4892dd2934d25b87f": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_5f35e41ac97a4f85a037d970351715a6",
       "IPY_MODEL_8912c9db83b5450aa9dec6b2571dc4a2"
      ],
      "layout": "IPY_MODEL_4ad144ee7d2545efbc2b33b5c48acdc4"
     }
    },
    "714f8746bc2b4ca7acb02f1e46ae2649": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "717db8fb535c40319b1c1f0d348c6f0d": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "7c77338ad5904b2da07e9917cf8bf2a6": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "84da667b57ee4dbd8cc29888a4ddcd84": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": "initial"
     }
    },
    "8715b7c80dd846d499d6f13f41c5b9ff": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "Downloading: 100%",
      "description_tooltip": null,
      "layout": "IPY_MODEL_e322f2029bc94a90ac48089627a52848",
      "max": 501200538,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_84da667b57ee4dbd8cc29888a4ddcd84",
      "value": 501200538
     }
    },
    "8912c9db83b5450aa9dec6b2571dc4a2": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_2d63cfdd601e4170afebd9ccede64520",
      "placeholder": "​",
      "style": "IPY_MODEL_ea9da948ded74743bd1a3d47a8e9d753",
      "value": " 456k/456k [00:00&lt;00:00, 1.12MB/s]"
     }
    },
    "8b7caa3fe3e54a60811d8d1091ef48c1": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "956c220b32b540f282d198583b19cd83": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_b5dbeb32d0824478a0e0bec4ee5e4834",
       "IPY_MODEL_130f67d003784be4b1178e2847f3527e"
      ],
      "layout": "IPY_MODEL_476732bced504081baa9d838b4e2ce1e"
     }
    },
    "9ab422d8880a48d3a26bfeca87c30f17": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_0dd0eb1c74eb447fbc6595dc7064405c",
       "IPY_MODEL_f201fc153dc64bed99019c35c751a6ab"
      ],
      "layout": "IPY_MODEL_bde57c476b1a40549280fd30b9c3580f"
     }
    },
    "a0d5af49e8534efdaf351c57f3a451ce": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "b5dbeb32d0824478a0e0bec4ee5e4834": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "Downloading: 100%",
      "description_tooltip": null,
      "layout": "IPY_MODEL_7c77338ad5904b2da07e9917cf8bf2a6",
      "max": 898823,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_259d27bec617486c9c146ea77f99b488",
      "value": 898823
     }
    },
    "bdb04ac4a9394c72a7f74c64e8dfe44c": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "bde57c476b1a40549280fd30b9c3580f": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "c2ee60b580c349cc8bb4612e159e4c34": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "cb4af4030f764ecdbd007d7555384294": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": "initial"
     }
    },
    "e322f2029bc94a90ac48089627a52848": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "ea9da948ded74743bd1a3d47a8e9d753": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "ed347becdda74efdb8290a2703867b17": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "f201fc153dc64bed99019c35c751a6ab": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_8b7caa3fe3e54a60811d8d1091ef48c1",
      "placeholder": "​",
      "style": "IPY_MODEL_ed347becdda74efdb8290a2703867b17",
      "value": " 481/481 [00:06&lt;00:00, 74.7B/s]"
     }
    },
    "f2ef9acb3545464caefb040ced4e70c0": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    }
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}
